hpcflow 0.1.15__py3-none-any.whl → 0.2.0a271__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/__init__.py +2 -11
- hpcflow/__pyinstaller/__init__.py +5 -0
- hpcflow/__pyinstaller/hook-hpcflow.py +40 -0
- hpcflow/_version.py +1 -1
- hpcflow/app.py +43 -0
- hpcflow/cli.py +2 -461
- hpcflow/data/demo_data_manifest/__init__.py +3 -0
- hpcflow/data/demo_data_manifest/demo_data_manifest.json +6 -0
- hpcflow/data/jinja_templates/test/test_template.txt +8 -0
- hpcflow/data/programs/hello_world/README.md +1 -0
- hpcflow/data/programs/hello_world/hello_world.c +87 -0
- hpcflow/data/programs/hello_world/linux/hello_world +0 -0
- hpcflow/data/programs/hello_world/macos/hello_world +0 -0
- hpcflow/data/programs/hello_world/win/hello_world.exe +0 -0
- hpcflow/data/scripts/__init__.py +1 -0
- hpcflow/data/scripts/bad_script.py +2 -0
- hpcflow/data/scripts/demo_task_1_generate_t1_infile_1.py +8 -0
- hpcflow/data/scripts/demo_task_1_generate_t1_infile_2.py +8 -0
- hpcflow/data/scripts/demo_task_1_parse_p3.py +7 -0
- hpcflow/data/scripts/do_nothing.py +2 -0
- hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py +4 -0
- hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py +8 -0
- hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py +4 -0
- hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py +4 -0
- hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py +7 -0
- hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py +4 -0
- hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py +7 -0
- hpcflow/data/scripts/generate_t1_file_01.py +7 -0
- hpcflow/data/scripts/import_future_script.py +7 -0
- hpcflow/data/scripts/input_file_generator_basic.py +3 -0
- hpcflow/data/scripts/input_file_generator_basic_FAIL.py +3 -0
- hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py +8 -0
- hpcflow/data/scripts/main_script_test_direct_in.py +3 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py +7 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_all_iters_test.py +15 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_env_spec.py +7 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_labels.py +8 -0
- hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_sub_param_in_direct_out.py +6 -0
- hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +12 -0
- hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py +12 -0
- hpcflow/data/scripts/main_script_test_hdf5_in_obj_group.py +12 -0
- hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +11 -0
- hpcflow/data/scripts/main_script_test_json_and_direct_in_json_out.py +14 -0
- hpcflow/data/scripts/main_script_test_json_in_json_and_direct_out.py +17 -0
- hpcflow/data/scripts/main_script_test_json_in_json_out.py +14 -0
- hpcflow/data/scripts/main_script_test_json_in_json_out_labels.py +16 -0
- hpcflow/data/scripts/main_script_test_json_in_obj.py +12 -0
- hpcflow/data/scripts/main_script_test_json_out_FAIL.py +3 -0
- hpcflow/data/scripts/main_script_test_json_out_obj.py +10 -0
- hpcflow/data/scripts/main_script_test_json_sub_param_in_json_out_labels.py +16 -0
- hpcflow/data/scripts/main_script_test_shell_env_vars.py +12 -0
- hpcflow/data/scripts/main_script_test_std_out_std_err.py +6 -0
- hpcflow/data/scripts/output_file_parser_basic.py +3 -0
- hpcflow/data/scripts/output_file_parser_basic_FAIL.py +7 -0
- hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py +8 -0
- hpcflow/data/scripts/parse_t1_file_01.py +4 -0
- hpcflow/data/scripts/script_exit_test.py +5 -0
- hpcflow/data/template_components/__init__.py +1 -0
- hpcflow/data/template_components/command_files.yaml +26 -0
- hpcflow/data/template_components/environments.yaml +13 -0
- hpcflow/data/template_components/parameters.yaml +14 -0
- hpcflow/data/template_components/task_schemas.yaml +139 -0
- hpcflow/data/workflows/workflow_1.yaml +5 -0
- hpcflow/examples.ipynb +1037 -0
- hpcflow/sdk/__init__.py +149 -0
- hpcflow/sdk/app.py +4266 -0
- hpcflow/sdk/cli.py +1479 -0
- hpcflow/sdk/cli_common.py +385 -0
- hpcflow/sdk/config/__init__.py +5 -0
- hpcflow/sdk/config/callbacks.py +246 -0
- hpcflow/sdk/config/cli.py +388 -0
- hpcflow/sdk/config/config.py +1410 -0
- hpcflow/sdk/config/config_file.py +501 -0
- hpcflow/sdk/config/errors.py +272 -0
- hpcflow/sdk/config/types.py +150 -0
- hpcflow/sdk/core/__init__.py +38 -0
- hpcflow/sdk/core/actions.py +3857 -0
- hpcflow/sdk/core/app_aware.py +25 -0
- hpcflow/sdk/core/cache.py +224 -0
- hpcflow/sdk/core/command_files.py +814 -0
- hpcflow/sdk/core/commands.py +424 -0
- hpcflow/sdk/core/element.py +2071 -0
- hpcflow/sdk/core/enums.py +221 -0
- hpcflow/sdk/core/environment.py +256 -0
- hpcflow/sdk/core/errors.py +1043 -0
- hpcflow/sdk/core/execute.py +207 -0
- hpcflow/sdk/core/json_like.py +809 -0
- hpcflow/sdk/core/loop.py +1320 -0
- hpcflow/sdk/core/loop_cache.py +282 -0
- hpcflow/sdk/core/object_list.py +933 -0
- hpcflow/sdk/core/parameters.py +3371 -0
- hpcflow/sdk/core/rule.py +196 -0
- hpcflow/sdk/core/run_dir_files.py +57 -0
- hpcflow/sdk/core/skip_reason.py +7 -0
- hpcflow/sdk/core/task.py +3792 -0
- hpcflow/sdk/core/task_schema.py +993 -0
- hpcflow/sdk/core/test_utils.py +538 -0
- hpcflow/sdk/core/types.py +447 -0
- hpcflow/sdk/core/utils.py +1207 -0
- hpcflow/sdk/core/validation.py +87 -0
- hpcflow/sdk/core/values.py +477 -0
- hpcflow/sdk/core/workflow.py +4820 -0
- hpcflow/sdk/core/zarr_io.py +206 -0
- hpcflow/sdk/data/__init__.py +13 -0
- hpcflow/sdk/data/config_file_schema.yaml +34 -0
- hpcflow/sdk/data/config_schema.yaml +260 -0
- hpcflow/sdk/data/environments_spec_schema.yaml +21 -0
- hpcflow/sdk/data/files_spec_schema.yaml +5 -0
- hpcflow/sdk/data/parameters_spec_schema.yaml +7 -0
- hpcflow/sdk/data/task_schema_spec_schema.yaml +3 -0
- hpcflow/sdk/data/workflow_spec_schema.yaml +22 -0
- hpcflow/sdk/demo/__init__.py +3 -0
- hpcflow/sdk/demo/cli.py +242 -0
- hpcflow/sdk/helper/__init__.py +3 -0
- hpcflow/sdk/helper/cli.py +137 -0
- hpcflow/sdk/helper/helper.py +300 -0
- hpcflow/sdk/helper/watcher.py +192 -0
- hpcflow/sdk/log.py +288 -0
- hpcflow/sdk/persistence/__init__.py +18 -0
- hpcflow/sdk/persistence/base.py +2817 -0
- hpcflow/sdk/persistence/defaults.py +6 -0
- hpcflow/sdk/persistence/discovery.py +39 -0
- hpcflow/sdk/persistence/json.py +954 -0
- hpcflow/sdk/persistence/pending.py +948 -0
- hpcflow/sdk/persistence/store_resource.py +203 -0
- hpcflow/sdk/persistence/types.py +309 -0
- hpcflow/sdk/persistence/utils.py +73 -0
- hpcflow/sdk/persistence/zarr.py +2388 -0
- hpcflow/sdk/runtime.py +320 -0
- hpcflow/sdk/submission/__init__.py +3 -0
- hpcflow/sdk/submission/enums.py +70 -0
- hpcflow/sdk/submission/jobscript.py +2379 -0
- hpcflow/sdk/submission/schedulers/__init__.py +281 -0
- hpcflow/sdk/submission/schedulers/direct.py +233 -0
- hpcflow/sdk/submission/schedulers/sge.py +376 -0
- hpcflow/sdk/submission/schedulers/slurm.py +598 -0
- hpcflow/sdk/submission/schedulers/utils.py +25 -0
- hpcflow/sdk/submission/shells/__init__.py +52 -0
- hpcflow/sdk/submission/shells/base.py +229 -0
- hpcflow/sdk/submission/shells/bash.py +504 -0
- hpcflow/sdk/submission/shells/os_version.py +115 -0
- hpcflow/sdk/submission/shells/powershell.py +352 -0
- hpcflow/sdk/submission/submission.py +1402 -0
- hpcflow/sdk/submission/types.py +140 -0
- hpcflow/sdk/typing.py +194 -0
- hpcflow/sdk/utils/arrays.py +69 -0
- hpcflow/sdk/utils/deferred_file.py +55 -0
- hpcflow/sdk/utils/hashing.py +16 -0
- hpcflow/sdk/utils/patches.py +31 -0
- hpcflow/sdk/utils/strings.py +69 -0
- hpcflow/tests/api/test_api.py +32 -0
- hpcflow/tests/conftest.py +123 -0
- hpcflow/tests/data/__init__.py +0 -0
- hpcflow/tests/data/benchmark_N_elements.yaml +6 -0
- hpcflow/tests/data/benchmark_script_runner.yaml +26 -0
- hpcflow/tests/data/multi_path_sequences.yaml +29 -0
- hpcflow/tests/data/workflow_1.json +10 -0
- hpcflow/tests/data/workflow_1.yaml +5 -0
- hpcflow/tests/data/workflow_1_slurm.yaml +8 -0
- hpcflow/tests/data/workflow_1_wsl.yaml +8 -0
- hpcflow/tests/data/workflow_test_run_abort.yaml +42 -0
- hpcflow/tests/jinja_templates/test_jinja_templates.py +161 -0
- hpcflow/tests/programs/test_programs.py +180 -0
- hpcflow/tests/schedulers/direct_linux/test_direct_linux_submission.py +12 -0
- hpcflow/tests/schedulers/sge/test_sge_submission.py +36 -0
- hpcflow/tests/schedulers/slurm/test_slurm_submission.py +14 -0
- hpcflow/tests/scripts/test_input_file_generators.py +282 -0
- hpcflow/tests/scripts/test_main_scripts.py +1361 -0
- hpcflow/tests/scripts/test_non_snippet_script.py +46 -0
- hpcflow/tests/scripts/test_ouput_file_parsers.py +353 -0
- hpcflow/tests/shells/wsl/test_wsl_submission.py +14 -0
- hpcflow/tests/unit/test_action.py +1066 -0
- hpcflow/tests/unit/test_action_rule.py +24 -0
- hpcflow/tests/unit/test_app.py +132 -0
- hpcflow/tests/unit/test_cache.py +46 -0
- hpcflow/tests/unit/test_cli.py +172 -0
- hpcflow/tests/unit/test_command.py +377 -0
- hpcflow/tests/unit/test_config.py +195 -0
- hpcflow/tests/unit/test_config_file.py +162 -0
- hpcflow/tests/unit/test_element.py +666 -0
- hpcflow/tests/unit/test_element_iteration.py +88 -0
- hpcflow/tests/unit/test_element_set.py +158 -0
- hpcflow/tests/unit/test_group.py +115 -0
- hpcflow/tests/unit/test_input_source.py +1479 -0
- hpcflow/tests/unit/test_input_value.py +398 -0
- hpcflow/tests/unit/test_jobscript_unit.py +757 -0
- hpcflow/tests/unit/test_json_like.py +1247 -0
- hpcflow/tests/unit/test_loop.py +2674 -0
- hpcflow/tests/unit/test_meta_task.py +325 -0
- hpcflow/tests/unit/test_multi_path_sequences.py +259 -0
- hpcflow/tests/unit/test_object_list.py +116 -0
- hpcflow/tests/unit/test_parameter.py +243 -0
- hpcflow/tests/unit/test_persistence.py +664 -0
- hpcflow/tests/unit/test_resources.py +243 -0
- hpcflow/tests/unit/test_run.py +286 -0
- hpcflow/tests/unit/test_run_directories.py +29 -0
- hpcflow/tests/unit/test_runtime.py +9 -0
- hpcflow/tests/unit/test_schema_input.py +372 -0
- hpcflow/tests/unit/test_shell.py +129 -0
- hpcflow/tests/unit/test_slurm.py +39 -0
- hpcflow/tests/unit/test_submission.py +502 -0
- hpcflow/tests/unit/test_task.py +2560 -0
- hpcflow/tests/unit/test_task_schema.py +182 -0
- hpcflow/tests/unit/test_utils.py +616 -0
- hpcflow/tests/unit/test_value_sequence.py +549 -0
- hpcflow/tests/unit/test_values.py +91 -0
- hpcflow/tests/unit/test_workflow.py +827 -0
- hpcflow/tests/unit/test_workflow_template.py +186 -0
- hpcflow/tests/unit/utils/test_arrays.py +40 -0
- hpcflow/tests/unit/utils/test_deferred_file_writer.py +34 -0
- hpcflow/tests/unit/utils/test_hashing.py +65 -0
- hpcflow/tests/unit/utils/test_patches.py +5 -0
- hpcflow/tests/unit/utils/test_redirect_std.py +50 -0
- hpcflow/tests/unit/utils/test_strings.py +97 -0
- hpcflow/tests/workflows/__init__.py +0 -0
- hpcflow/tests/workflows/test_directory_structure.py +31 -0
- hpcflow/tests/workflows/test_jobscript.py +355 -0
- hpcflow/tests/workflows/test_run_status.py +198 -0
- hpcflow/tests/workflows/test_skip_downstream.py +696 -0
- hpcflow/tests/workflows/test_submission.py +140 -0
- hpcflow/tests/workflows/test_workflows.py +564 -0
- hpcflow/tests/workflows/test_zip.py +18 -0
- hpcflow/viz_demo.ipynb +6794 -0
- hpcflow-0.2.0a271.dist-info/LICENSE +375 -0
- hpcflow-0.2.0a271.dist-info/METADATA +65 -0
- hpcflow-0.2.0a271.dist-info/RECORD +237 -0
- {hpcflow-0.1.15.dist-info → hpcflow-0.2.0a271.dist-info}/WHEEL +4 -5
- hpcflow-0.2.0a271.dist-info/entry_points.txt +6 -0
- hpcflow/api.py +0 -490
- hpcflow/archive/archive.py +0 -307
- hpcflow/archive/cloud/cloud.py +0 -45
- hpcflow/archive/cloud/errors.py +0 -9
- hpcflow/archive/cloud/providers/dropbox.py +0 -427
- hpcflow/archive/errors.py +0 -5
- hpcflow/base_db.py +0 -4
- hpcflow/config.py +0 -233
- hpcflow/copytree.py +0 -66
- hpcflow/data/examples/_config.yml +0 -14
- hpcflow/data/examples/damask/demo/1.run.yml +0 -4
- hpcflow/data/examples/damask/demo/2.process.yml +0 -29
- hpcflow/data/examples/damask/demo/geom.geom +0 -2052
- hpcflow/data/examples/damask/demo/load.load +0 -1
- hpcflow/data/examples/damask/demo/material.config +0 -185
- hpcflow/data/examples/damask/inputs/geom.geom +0 -2052
- hpcflow/data/examples/damask/inputs/load.load +0 -1
- hpcflow/data/examples/damask/inputs/material.config +0 -185
- hpcflow/data/examples/damask/profiles/_variable_lookup.yml +0 -21
- hpcflow/data/examples/damask/profiles/damask.yml +0 -4
- hpcflow/data/examples/damask/profiles/damask_process.yml +0 -8
- hpcflow/data/examples/damask/profiles/damask_run.yml +0 -5
- hpcflow/data/examples/damask/profiles/default.yml +0 -6
- hpcflow/data/examples/thinking.yml +0 -177
- hpcflow/errors.py +0 -2
- hpcflow/init_db.py +0 -37
- hpcflow/models.py +0 -2595
- hpcflow/nesting.py +0 -9
- hpcflow/profiles.py +0 -455
- hpcflow/project.py +0 -81
- hpcflow/scheduler.py +0 -322
- hpcflow/utils.py +0 -103
- hpcflow/validation.py +0 -166
- hpcflow/variables.py +0 -543
- hpcflow-0.1.15.dist-info/METADATA +0 -168
- hpcflow-0.1.15.dist-info/RECORD +0 -45
- hpcflow-0.1.15.dist-info/entry_points.txt +0 -8
- hpcflow-0.1.15.dist-info/top_level.txt +0 -1
- /hpcflow/{archive → data/jinja_templates}/__init__.py +0 -0
- /hpcflow/{archive/cloud → data/programs}/__init__.py +0 -0
- /hpcflow/{archive/cloud/providers → data/workflows}/__init__.py +0 -0
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
from textwrap import dedent
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from hpcflow.app import app as hf
|
|
6
|
+
from hpcflow.sdk.config.errors import UnknownMetaTaskConstitutiveSchema
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_basic_meta_task_workflow(new_null_config, tmp_path):
|
|
10
|
+
wk_yaml = dedent(
|
|
11
|
+
"""\
|
|
12
|
+
name: test_meta_task
|
|
13
|
+
template_components:
|
|
14
|
+
task_schemas:
|
|
15
|
+
- objective: s0
|
|
16
|
+
inputs:
|
|
17
|
+
- parameter: p1
|
|
18
|
+
outputs:
|
|
19
|
+
- parameter: p2
|
|
20
|
+
actions:
|
|
21
|
+
- commands:
|
|
22
|
+
- command: echo "$((<<parameter:p1>> + 1))"
|
|
23
|
+
stdout: <<int(parameter:p2)>>
|
|
24
|
+
|
|
25
|
+
- objective: s1
|
|
26
|
+
inputs:
|
|
27
|
+
- parameter: p2
|
|
28
|
+
- parameter: p2b
|
|
29
|
+
outputs:
|
|
30
|
+
- parameter: p3
|
|
31
|
+
actions:
|
|
32
|
+
- commands:
|
|
33
|
+
- command: echo "$((<<parameter:p2>> + <<parameter:p2b>>))"
|
|
34
|
+
stdout: <<int(parameter:p3)>>
|
|
35
|
+
|
|
36
|
+
- objective: s2
|
|
37
|
+
inputs:
|
|
38
|
+
- parameter: p3
|
|
39
|
+
outputs:
|
|
40
|
+
- parameter: p4
|
|
41
|
+
actions:
|
|
42
|
+
- commands:
|
|
43
|
+
- command: echo "$((<<parameter:p3>> + 1))"
|
|
44
|
+
stdout: <<int(parameter:p4)>>
|
|
45
|
+
|
|
46
|
+
- objective: s3
|
|
47
|
+
inputs:
|
|
48
|
+
- parameter: p4
|
|
49
|
+
outputs:
|
|
50
|
+
- parameter: p5
|
|
51
|
+
actions:
|
|
52
|
+
- commands:
|
|
53
|
+
- command: echo "$((<<parameter:p4>> + 1))"
|
|
54
|
+
stdout: <<int(parameter:p5)>>
|
|
55
|
+
|
|
56
|
+
meta_task_schemas:
|
|
57
|
+
- objective: system_analysis
|
|
58
|
+
inputs:
|
|
59
|
+
- parameter: p2
|
|
60
|
+
outputs:
|
|
61
|
+
- parameter: p4
|
|
62
|
+
|
|
63
|
+
meta_tasks:
|
|
64
|
+
system_analysis:
|
|
65
|
+
- schema: s1
|
|
66
|
+
inputs:
|
|
67
|
+
p2b: 220
|
|
68
|
+
- schema: s2
|
|
69
|
+
|
|
70
|
+
tasks:
|
|
71
|
+
- schema: s0
|
|
72
|
+
inputs:
|
|
73
|
+
p1: 100
|
|
74
|
+
- schema: system_analysis
|
|
75
|
+
- schema: s3
|
|
76
|
+
"""
|
|
77
|
+
)
|
|
78
|
+
wk = hf.Workflow.from_YAML_string(wk_yaml, path=tmp_path)
|
|
79
|
+
|
|
80
|
+
# basic check of param dependendices
|
|
81
|
+
s0_di = wk.tasks.s0.elements[0].get_data_idx()
|
|
82
|
+
s1_di = wk.tasks.s1.elements[0].get_data_idx()
|
|
83
|
+
s2_di = wk.tasks.s2.elements[0].get_data_idx()
|
|
84
|
+
s3_di = wk.tasks.s3.elements[0].get_data_idx()
|
|
85
|
+
|
|
86
|
+
assert s0_di["outputs.p2"] == s1_di["inputs.p2"]
|
|
87
|
+
assert s1_di["outputs.p3"] == s2_di["inputs.p3"]
|
|
88
|
+
assert s2_di["outputs.p4"] == s3_di["inputs.p4"]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_basic_meta_task_workflow_API(new_null_config, tmp_path):
|
|
92
|
+
"""as above but using Python API."""
|
|
93
|
+
# normal task schemas:
|
|
94
|
+
s0 = hf.TaskSchema(
|
|
95
|
+
objective="s0",
|
|
96
|
+
inputs=[hf.SchemaInput("p1")],
|
|
97
|
+
outputs=[hf.SchemaOutput("p2")],
|
|
98
|
+
actions=[
|
|
99
|
+
hf.Action(
|
|
100
|
+
commands=[
|
|
101
|
+
hf.Command(
|
|
102
|
+
command='echo "$((<<parameter:p1>> + 1))"',
|
|
103
|
+
stdout="<<int(parameter:p2)>>",
|
|
104
|
+
)
|
|
105
|
+
]
|
|
106
|
+
)
|
|
107
|
+
],
|
|
108
|
+
)
|
|
109
|
+
s1 = hf.TaskSchema(
|
|
110
|
+
objective="s1",
|
|
111
|
+
inputs=[hf.SchemaInput("p2")],
|
|
112
|
+
outputs=[hf.SchemaOutput("p3")],
|
|
113
|
+
actions=[
|
|
114
|
+
hf.Action(
|
|
115
|
+
commands=[
|
|
116
|
+
hf.Command(
|
|
117
|
+
command='echo "$((<<parameter:p2>> + 1))"',
|
|
118
|
+
stdout="<<int(parameter:p3)>>",
|
|
119
|
+
)
|
|
120
|
+
]
|
|
121
|
+
)
|
|
122
|
+
],
|
|
123
|
+
)
|
|
124
|
+
s2 = hf.TaskSchema(
|
|
125
|
+
objective="s2",
|
|
126
|
+
inputs=[hf.SchemaInput("p3")],
|
|
127
|
+
outputs=[hf.SchemaOutput("p4")],
|
|
128
|
+
actions=[
|
|
129
|
+
hf.Action(
|
|
130
|
+
commands=[
|
|
131
|
+
hf.Command(
|
|
132
|
+
command='echo "$((<<parameter:p3>> + 1))"',
|
|
133
|
+
stdout="<<int(parameter:p4)>>",
|
|
134
|
+
)
|
|
135
|
+
]
|
|
136
|
+
)
|
|
137
|
+
],
|
|
138
|
+
)
|
|
139
|
+
s3 = hf.TaskSchema(
|
|
140
|
+
objective="s3",
|
|
141
|
+
inputs=[hf.SchemaInput("p4")],
|
|
142
|
+
outputs=[hf.SchemaOutput("p5")],
|
|
143
|
+
actions=[
|
|
144
|
+
hf.Action(
|
|
145
|
+
commands=[
|
|
146
|
+
hf.Command(
|
|
147
|
+
command='echo "$((<<parameter:p4>> + 1))"',
|
|
148
|
+
stdout="<<int(parameter:p5)>>",
|
|
149
|
+
)
|
|
150
|
+
]
|
|
151
|
+
)
|
|
152
|
+
],
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
# meta=task schema:
|
|
156
|
+
ms = hf.MetaTaskSchema(
|
|
157
|
+
objective="system_analysis",
|
|
158
|
+
inputs=[hf.SchemaInput("p2")],
|
|
159
|
+
outputs=[hf.SchemaOutput("p4")],
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
# meta-task:
|
|
163
|
+
m1 = hf.MetaTask(
|
|
164
|
+
schema=ms,
|
|
165
|
+
tasks=[
|
|
166
|
+
hf.Task(schema=s1),
|
|
167
|
+
hf.Task(schema=s2),
|
|
168
|
+
],
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
# workflow template tasks list:
|
|
172
|
+
tasks = [
|
|
173
|
+
hf.Task(schema=s0, inputs={"p1": 100}),
|
|
174
|
+
m1,
|
|
175
|
+
hf.Task(schema=s3),
|
|
176
|
+
]
|
|
177
|
+
|
|
178
|
+
wk = hf.Workflow.from_template_data(
|
|
179
|
+
template_name="meta_task_workflow",
|
|
180
|
+
tasks=tasks,
|
|
181
|
+
path=tmp_path,
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
# basic check of param dependendices
|
|
185
|
+
s0_di = wk.tasks.s0.elements[0].get_data_idx()
|
|
186
|
+
s1_di = wk.tasks.s1.elements[0].get_data_idx()
|
|
187
|
+
s2_di = wk.tasks.s2.elements[0].get_data_idx()
|
|
188
|
+
s3_di = wk.tasks.s3.elements[0].get_data_idx()
|
|
189
|
+
|
|
190
|
+
assert s0_di["outputs.p2"] == s1_di["inputs.p2"]
|
|
191
|
+
assert s1_di["outputs.p3"] == s2_di["inputs.p3"]
|
|
192
|
+
assert s2_di["outputs.p4"] == s3_di["inputs.p4"]
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def test_meta_task_custom_parametrisation(new_null_config, tmp_path):
|
|
196
|
+
"""test customising the parametrisation of inputs, sequences, and resources within the
|
|
197
|
+
`tasks` list."""
|
|
198
|
+
wk_yaml = dedent(
|
|
199
|
+
"""\
|
|
200
|
+
name: test_metatask_multi_element_sets_custom_parametrisation
|
|
201
|
+
template_components:
|
|
202
|
+
task_schemas:
|
|
203
|
+
- objective: s1
|
|
204
|
+
inputs:
|
|
205
|
+
- parameter: p1
|
|
206
|
+
- parameter: p2
|
|
207
|
+
outputs:
|
|
208
|
+
- parameter: p3
|
|
209
|
+
actions:
|
|
210
|
+
- commands:
|
|
211
|
+
- command: echo "$((<<parameter:p1>> + <<parameter:p2>>))"
|
|
212
|
+
stdout: <<int(parameter:p3)>>
|
|
213
|
+
|
|
214
|
+
meta_task_schemas:
|
|
215
|
+
- objective: system_analysis
|
|
216
|
+
inputs:
|
|
217
|
+
- parameter: p1
|
|
218
|
+
- parameter: p2
|
|
219
|
+
outputs:
|
|
220
|
+
- parameter: p3
|
|
221
|
+
|
|
222
|
+
meta_tasks:
|
|
223
|
+
system_analysis:
|
|
224
|
+
- schema: s1
|
|
225
|
+
element_sets:
|
|
226
|
+
- inputs:
|
|
227
|
+
p1: 100
|
|
228
|
+
p2: 200
|
|
229
|
+
- inputs:
|
|
230
|
+
p1: 100
|
|
231
|
+
sequences:
|
|
232
|
+
- path: inputs.p2
|
|
233
|
+
values: [200, 201]
|
|
234
|
+
tasks:
|
|
235
|
+
- schema: system_analysis
|
|
236
|
+
inputs:
|
|
237
|
+
s1: # should apply to first element set by default
|
|
238
|
+
p1: 101
|
|
239
|
+
resources:
|
|
240
|
+
s1: # should apply to first element set by default
|
|
241
|
+
any:
|
|
242
|
+
num_cores: 2
|
|
243
|
+
- schema: system_analysis
|
|
244
|
+
inputs:
|
|
245
|
+
s1.0: # applies to first element set of s1
|
|
246
|
+
p1: 102
|
|
247
|
+
s1.1: # applies to second element set of s1
|
|
248
|
+
p1: 103
|
|
249
|
+
sequences:
|
|
250
|
+
s1.1: # sequences list in second element set is replaced with this list:
|
|
251
|
+
- path: inputs.p2
|
|
252
|
+
values: [300, 301]
|
|
253
|
+
"""
|
|
254
|
+
)
|
|
255
|
+
wk = hf.Workflow.from_YAML_string(wk_yaml, path=tmp_path)
|
|
256
|
+
|
|
257
|
+
assert wk.tasks.s1_1.template.element_sets[0].resources[0].num_cores == 2 # modified
|
|
258
|
+
assert (
|
|
259
|
+
wk.tasks.s1_2.template.element_sets[0].resources[0].num_cores is None
|
|
260
|
+
) # unaffected
|
|
261
|
+
|
|
262
|
+
assert wk.tasks.s1_1.template.element_sets[0].inputs[0].value == 101 # modified
|
|
263
|
+
assert wk.tasks.s1_1.template.element_sets[0].inputs[1].value == 200 # unaffected
|
|
264
|
+
assert wk.tasks.s1_1.template.element_sets[1].sequences[0].values == [
|
|
265
|
+
200,
|
|
266
|
+
201,
|
|
267
|
+
] # unaffected
|
|
268
|
+
|
|
269
|
+
assert wk.tasks.s1_2.template.element_sets[0].inputs[0].value == 102 # modified
|
|
270
|
+
assert wk.tasks.s1_2.template.element_sets[1].inputs[0].value == 103 # modified
|
|
271
|
+
assert wk.tasks.s1_2.template.element_sets[1].sequences[0].values == [
|
|
272
|
+
300,
|
|
273
|
+
301,
|
|
274
|
+
] # modified
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def test_meta_task_custom_parametrisation_raises_on_bad_schema_name(
|
|
278
|
+
new_null_config, tmp_path
|
|
279
|
+
):
|
|
280
|
+
wk_yaml = dedent(
|
|
281
|
+
"""\
|
|
282
|
+
name: test_metatask_raise_on_bad_schema_name
|
|
283
|
+
template_components:
|
|
284
|
+
task_schemas:
|
|
285
|
+
- objective: s1
|
|
286
|
+
inputs:
|
|
287
|
+
- parameter: p1
|
|
288
|
+
- parameter: p2
|
|
289
|
+
outputs:
|
|
290
|
+
- parameter: p3
|
|
291
|
+
actions:
|
|
292
|
+
- commands:
|
|
293
|
+
- command: echo "$((<<parameter:p1>> + <<parameter:p2>>))"
|
|
294
|
+
stdout: <<int(parameter:p3)>>
|
|
295
|
+
|
|
296
|
+
meta_task_schemas:
|
|
297
|
+
- objective: system_analysis
|
|
298
|
+
inputs:
|
|
299
|
+
- parameter: p1
|
|
300
|
+
- parameter: p2
|
|
301
|
+
outputs:
|
|
302
|
+
- parameter: p3
|
|
303
|
+
|
|
304
|
+
meta_tasks:
|
|
305
|
+
system_analysis:
|
|
306
|
+
- schema: s1
|
|
307
|
+
element_sets:
|
|
308
|
+
- inputs:
|
|
309
|
+
p1: 100
|
|
310
|
+
p2: 200
|
|
311
|
+
- inputs:
|
|
312
|
+
p1: 100
|
|
313
|
+
sequences:
|
|
314
|
+
- path: inputs.p2
|
|
315
|
+
values: [200, 201]
|
|
316
|
+
tasks:
|
|
317
|
+
- schema: system_analysis
|
|
318
|
+
resources:
|
|
319
|
+
BAD_SCHEMA_NAME: # should raise!
|
|
320
|
+
any:
|
|
321
|
+
num_cores: 2
|
|
322
|
+
"""
|
|
323
|
+
)
|
|
324
|
+
with pytest.raises(UnknownMetaTaskConstitutiveSchema):
|
|
325
|
+
wk = hf.Workflow.from_YAML_string(wk_yaml, path=tmp_path)
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from textwrap import dedent
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from hpcflow.app import app as hf
|
|
8
|
+
from hpcflow.sdk.core.test_utils import make_schemas
|
|
9
|
+
from hpcflow.sdk.core.utils import get_file_context
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_MPS_sequences():
|
|
13
|
+
mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
|
|
14
|
+
assert mps.sequences == [
|
|
15
|
+
hf.ValueSequence(path="inputs.p1", values=[0, 1]),
|
|
16
|
+
hf.ValueSequence(path="inputs.p2", values=[2, 3]),
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_MPS_sequences_moved_to_element_set():
|
|
21
|
+
mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
|
|
22
|
+
es = hf.ElementSet(multi_path_sequences=[mps])
|
|
23
|
+
expected_mps_seqs = [
|
|
24
|
+
hf.ValueSequence(path="inputs.p1", values=[0, 1]),
|
|
25
|
+
hf.ValueSequence(path="inputs.p2", values=[2, 3]),
|
|
26
|
+
]
|
|
27
|
+
assert mps._sequences is None
|
|
28
|
+
assert mps.sequence_indices == [0, 2]
|
|
29
|
+
assert es.sequences == mps.sequences
|
|
30
|
+
assert es.sequences == expected_mps_seqs
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_MPS_sequences_moved_to_element_set_with_existing_sequences():
|
|
34
|
+
mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
|
|
35
|
+
seq = hf.ValueSequence(path="inputs.p0", values=[0, 1])
|
|
36
|
+
expected_mps_seqs = [
|
|
37
|
+
hf.ValueSequence(path="inputs.p1", values=[0, 1]),
|
|
38
|
+
hf.ValueSequence(path="inputs.p2", values=[2, 3]),
|
|
39
|
+
]
|
|
40
|
+
es = hf.ElementSet(
|
|
41
|
+
sequences=[seq],
|
|
42
|
+
multi_path_sequences=[mps],
|
|
43
|
+
)
|
|
44
|
+
assert mps._sequences is None
|
|
45
|
+
assert mps.sequence_indices == [1, 3]
|
|
46
|
+
assert mps.sequences == expected_mps_seqs
|
|
47
|
+
assert es.sequences == [seq, *expected_mps_seqs]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_MPS_sequences_moved_to_task_element_set():
|
|
51
|
+
mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
|
|
52
|
+
s1 = make_schemas(({"p1": None, "p2": None}, ()))
|
|
53
|
+
t1 = hf.Task(s1, multi_path_sequences=[mps])
|
|
54
|
+
expected_mps_seqs = [
|
|
55
|
+
hf.ValueSequence(path="inputs.p1", values=[0, 1]),
|
|
56
|
+
hf.ValueSequence(path="inputs.p2", values=[2, 3]),
|
|
57
|
+
]
|
|
58
|
+
es = t1.element_sets[0]
|
|
59
|
+
assert mps._sequences is None
|
|
60
|
+
assert mps.sequence_indices == [0, 2]
|
|
61
|
+
assert es.sequences == mps.sequences
|
|
62
|
+
assert es.sequences == expected_mps_seqs
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_MPS_sequences_moved_to_task_element_set_with_existing_sequences():
|
|
66
|
+
mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
|
|
67
|
+
seq = hf.ValueSequence(path="inputs.p0", values=[0, 1])
|
|
68
|
+
s1 = make_schemas(({"p0": None, "p1": None, "p2": None}, ()))
|
|
69
|
+
t1 = hf.Task(s1, sequences=[seq], multi_path_sequences=[mps])
|
|
70
|
+
expected_mps_seqs = [
|
|
71
|
+
hf.ValueSequence(path="inputs.p1", values=[0, 1]),
|
|
72
|
+
hf.ValueSequence(path="inputs.p2", values=[2, 3]),
|
|
73
|
+
]
|
|
74
|
+
es = t1.element_sets[0]
|
|
75
|
+
assert mps._sequences is None
|
|
76
|
+
assert mps.sequence_indices == [1, 3]
|
|
77
|
+
assert mps.sequences == expected_mps_seqs
|
|
78
|
+
assert es.sequences == [seq, *expected_mps_seqs]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_MPS_sequence_element_inputs(null_config, tmp_path):
|
|
82
|
+
mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
|
|
83
|
+
s1 = make_schemas(({"p1": None, "p2": None}, ()))
|
|
84
|
+
t1 = hf.Task(s1, multi_path_sequences=[mps])
|
|
85
|
+
wf = hf.Workflow.from_template_data(
|
|
86
|
+
tasks=[t1],
|
|
87
|
+
template_name="test_multi_path_sequence",
|
|
88
|
+
path=tmp_path,
|
|
89
|
+
)
|
|
90
|
+
assert len(wf.template.tasks[0].element_sets[0].sequences) == 2
|
|
91
|
+
assert wf.tasks[0].num_elements == 2
|
|
92
|
+
assert wf.tasks[0].elements[0].get("inputs") == {"p1": 0, "p2": 2}
|
|
93
|
+
assert wf.tasks[0].elements[1].get("inputs") == {"p1": 1, "p2": 3}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_MPS_sequence_element_inputs_with_existing_sequence(null_config, tmp_path):
|
|
97
|
+
mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
|
|
98
|
+
seq = hf.ValueSequence(path="inputs.p0", values=[0, 1])
|
|
99
|
+
s1 = make_schemas(({"p0": None, "p1": None, "p2": None}, ()))
|
|
100
|
+
t1 = hf.Task(s1, sequences=[seq], multi_path_sequences=[mps])
|
|
101
|
+
wf = hf.Workflow.from_template_data(
|
|
102
|
+
tasks=[t1],
|
|
103
|
+
template_name="test_multi_path_sequence",
|
|
104
|
+
path=tmp_path,
|
|
105
|
+
)
|
|
106
|
+
assert len(wf.template.tasks[0].element_sets[0].sequences) == 3
|
|
107
|
+
assert wf.tasks[0].num_elements == 2
|
|
108
|
+
assert wf.tasks[0].elements[0].get("inputs") == {"p0": 0, "p1": 0, "p2": 2}
|
|
109
|
+
assert wf.tasks[0].elements[1].get("inputs") == {"p0": 1, "p1": 1, "p2": 3}
|
|
110
|
+
|
|
111
|
+
# check the same on reload:
|
|
112
|
+
wf = wf.reload()
|
|
113
|
+
assert len(wf.template.tasks[0].element_sets[0].sequences) == 3
|
|
114
|
+
assert wf.tasks[0].num_elements == 2
|
|
115
|
+
assert wf.tasks[0].elements[0].get("inputs") == {"p0": 0, "p1": 0, "p2": 2}
|
|
116
|
+
assert wf.tasks[0].elements[1].get("inputs") == {"p0": 1, "p1": 1, "p2": 3}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@pytest.mark.integration
|
|
120
|
+
def test_MPS_element_outputs(null_config, tmp_path):
|
|
121
|
+
with get_file_context("hpcflow.tests.data", "multi_path_sequences.yaml") as file_path:
|
|
122
|
+
wf = hf.make_and_submit_workflow(
|
|
123
|
+
file_path,
|
|
124
|
+
path=tmp_path,
|
|
125
|
+
status=False,
|
|
126
|
+
add_to_known=False,
|
|
127
|
+
wait=True,
|
|
128
|
+
)
|
|
129
|
+
assert wf.tasks[0].num_elements == 2
|
|
130
|
+
|
|
131
|
+
p2 = wf.tasks[0].elements[0].outputs.p2
|
|
132
|
+
assert isinstance(p2, hf.ElementParameter)
|
|
133
|
+
assert p2.value == 302
|
|
134
|
+
|
|
135
|
+
p2 = wf.tasks[0].elements[1].outputs.p2
|
|
136
|
+
assert isinstance(p2, hf.ElementParameter)
|
|
137
|
+
assert p2.value == 304
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def test_MPS_latin_hypercube_sequence_values():
|
|
141
|
+
wft_yaml = dedent(
|
|
142
|
+
"""\
|
|
143
|
+
name: test_latin_hypercube_sampling
|
|
144
|
+
template_components:
|
|
145
|
+
task_schemas:
|
|
146
|
+
- objective: define_p1
|
|
147
|
+
inputs:
|
|
148
|
+
- parameter: p1
|
|
149
|
+
tasks:
|
|
150
|
+
- schema: define_p1
|
|
151
|
+
inputs:
|
|
152
|
+
p1: {}
|
|
153
|
+
multi_path_sequences:
|
|
154
|
+
- paths: [inputs.p1.a, inputs.p1.b]
|
|
155
|
+
values::from_latin_hypercube:
|
|
156
|
+
num_samples: 5
|
|
157
|
+
"""
|
|
158
|
+
)
|
|
159
|
+
wft = hf.WorkflowTemplate.from_YAML_string(wft_yaml)
|
|
160
|
+
es = wft.tasks[0].element_sets[0]
|
|
161
|
+
assert len(es.multi_path_sequences) == 1
|
|
162
|
+
mps_values = np.asarray(es.multi_path_sequences[0].values)
|
|
163
|
+
assert mps_values.shape == (2, 5)
|
|
164
|
+
assert len(es.sequences) == 2
|
|
165
|
+
seq_1 = es.sequences[0]
|
|
166
|
+
seq_2 = es.sequences[1]
|
|
167
|
+
assert seq_1.path == "inputs.p1.a"
|
|
168
|
+
assert seq_2.path == "inputs.p1.b"
|
|
169
|
+
assert np.array_equal(np.asarray(seq_1.values), mps_values[0])
|
|
170
|
+
assert np.array_equal(np.asarray(seq_2.values), mps_values[1])
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def test_MPS_latin_hypercube_sequence_bounds():
|
|
174
|
+
|
|
175
|
+
bounds = {
|
|
176
|
+
"inputs.a": {"extent": [16789.2, 17812.5], "scaling": "linear"},
|
|
177
|
+
"inputs.c": {"extent": [1.0e-10, 1.0e-5], "scaling": "log"},
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
mps = hf.MultiPathSequence.from_latin_hypercube(
|
|
181
|
+
paths=["inputs.a", "inputs.b", "inputs.c"],
|
|
182
|
+
num_samples=10,
|
|
183
|
+
bounds=bounds,
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
vals_arr = np.array(mps.values)
|
|
187
|
+
|
|
188
|
+
assert vals_arr.shape == (3, 10)
|
|
189
|
+
|
|
190
|
+
vals_a = vals_arr[0]
|
|
191
|
+
vals_b = vals_arr[1]
|
|
192
|
+
vals_c = vals_arr[2]
|
|
193
|
+
|
|
194
|
+
extent_a = bounds["inputs.a"]["extent"]
|
|
195
|
+
extent_b = [0.0, 1.0]
|
|
196
|
+
extent_c = bounds["inputs.c"]["extent"]
|
|
197
|
+
|
|
198
|
+
assert np.logical_and(vals_a > extent_a[0], vals_a < extent_a[1]).all()
|
|
199
|
+
assert np.logical_and(vals_b > extent_b[0], vals_b < extent_b[1]).all()
|
|
200
|
+
assert np.logical_and(vals_c > extent_c[0], vals_c < extent_c[1]).all()
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def test_MPS_move_from_sequences_list():
|
|
204
|
+
wft_yaml = dedent(
|
|
205
|
+
"""\
|
|
206
|
+
name: test_latin_hypercube_sampling
|
|
207
|
+
template_components:
|
|
208
|
+
task_schemas:
|
|
209
|
+
- objective: define_p1_p2_p3_p4
|
|
210
|
+
inputs:
|
|
211
|
+
- parameter: p1
|
|
212
|
+
- parameter: p2
|
|
213
|
+
- parameter: p3
|
|
214
|
+
- parameter: p4
|
|
215
|
+
tasks:
|
|
216
|
+
- schema: define_p1_p2_p3_p4
|
|
217
|
+
inputs:
|
|
218
|
+
p1: {}
|
|
219
|
+
p2: {}
|
|
220
|
+
p3: {}
|
|
221
|
+
|
|
222
|
+
multi_path_sequences:
|
|
223
|
+
- paths: [inputs.p1.a, inputs.p1.b]
|
|
224
|
+
values::from_latin_hypercube:
|
|
225
|
+
num_samples: 4
|
|
226
|
+
|
|
227
|
+
sequences:
|
|
228
|
+
- paths: [inputs.p2.a, inputs.p2.b] # actually a multi-path sequence
|
|
229
|
+
values::from_latin_hypercube:
|
|
230
|
+
num_samples: 4
|
|
231
|
+
|
|
232
|
+
- path: inputs.p4 # a normal sequence
|
|
233
|
+
values: [0, 1, 2, 3]
|
|
234
|
+
|
|
235
|
+
- paths: [inputs.p3.a, inputs.p3.b] # actually a multi-path sequence
|
|
236
|
+
values::from_latin_hypercube:
|
|
237
|
+
num_samples: 4
|
|
238
|
+
"""
|
|
239
|
+
)
|
|
240
|
+
wft = hf.WorkflowTemplate.from_YAML_string(wft_yaml)
|
|
241
|
+
es = wft.tasks[0].element_sets[0]
|
|
242
|
+
mps_lst = es.multi_path_sequences
|
|
243
|
+
seq_lst = es.sequences
|
|
244
|
+
assert len(mps_lst) == 3
|
|
245
|
+
assert len(seq_lst) == 7 # one original plus three multi-path with two paths each
|
|
246
|
+
|
|
247
|
+
# check ordering of multi-path sequences is preserved:
|
|
248
|
+
assert mps_lst[0].paths == ["inputs.p1.a", "inputs.p1.b"]
|
|
249
|
+
assert mps_lst[1].paths == ["inputs.p2.a", "inputs.p2.b"]
|
|
250
|
+
assert mps_lst[2].paths == ["inputs.p3.a", "inputs.p3.b"]
|
|
251
|
+
|
|
252
|
+
# check sensible ordering of sequences:
|
|
253
|
+
assert seq_lst[0].path == "inputs.p4"
|
|
254
|
+
assert seq_lst[1].path == "inputs.p1.a"
|
|
255
|
+
assert seq_lst[2].path == "inputs.p1.b"
|
|
256
|
+
assert seq_lst[3].path == "inputs.p2.a"
|
|
257
|
+
assert seq_lst[4].path == "inputs.p2.b"
|
|
258
|
+
assert seq_lst[5].path == "inputs.p3.a"
|
|
259
|
+
assert seq_lst[6].path == "inputs.p3.b"
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing_extensions import TypedDict
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from hpcflow.app import app as hf
|
|
7
|
+
from hpcflow.sdk.core.object_list import ObjectList, DotAccessObjectList
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.fixture
|
|
11
|
+
def null_config(tmp_path):
|
|
12
|
+
if not hf.is_config_loaded:
|
|
13
|
+
hf.load_config(config_dir=tmp_path)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class MyObj:
|
|
18
|
+
name: str
|
|
19
|
+
data: int
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SimpleObjectList(TypedDict):
|
|
23
|
+
objects: list[MyObj]
|
|
24
|
+
object_list: DotAccessObjectList
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@pytest.fixture
|
|
28
|
+
def simple_object_list(null_config) -> SimpleObjectList:
|
|
29
|
+
my_objs = [MyObj(name="A", data=1), MyObj(name="B", data=2)]
|
|
30
|
+
obj_list = DotAccessObjectList(my_objs, access_attribute="name")
|
|
31
|
+
return {"objects": my_objs, "object_list": obj_list}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_get_item(simple_object_list: SimpleObjectList):
|
|
35
|
+
objects = simple_object_list["objects"]
|
|
36
|
+
obj_list = simple_object_list["object_list"]
|
|
37
|
+
|
|
38
|
+
assert obj_list[0] == objects[0] and obj_list[1] == objects[1]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_get_dot_notation(simple_object_list: SimpleObjectList):
|
|
42
|
+
objects = simple_object_list["objects"]
|
|
43
|
+
obj_list = simple_object_list["object_list"]
|
|
44
|
+
|
|
45
|
+
assert obj_list.A == objects[0] and obj_list.B == objects[1]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_add_obj_to_end(simple_object_list: SimpleObjectList):
|
|
49
|
+
obj_list = simple_object_list["object_list"]
|
|
50
|
+
new_obj = MyObj("C", 3)
|
|
51
|
+
obj_list.add_object(new_obj)
|
|
52
|
+
assert obj_list[-1] == new_obj
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_add_obj_to_start(simple_object_list: SimpleObjectList):
|
|
56
|
+
obj_list = simple_object_list["object_list"]
|
|
57
|
+
new_obj = MyObj("C", 3)
|
|
58
|
+
obj_list.add_object(new_obj, 0)
|
|
59
|
+
assert obj_list[0] == new_obj
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_add_obj_to_middle(simple_object_list: SimpleObjectList):
|
|
63
|
+
obj_list = simple_object_list["object_list"]
|
|
64
|
+
new_obj = MyObj("C", 3)
|
|
65
|
+
obj_list.add_object(new_obj, 1)
|
|
66
|
+
assert obj_list[1] == new_obj
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_get_obj_attr_custom_callable(null_config):
|
|
70
|
+
def my_get_obj_attr(self, obj, attr):
|
|
71
|
+
if attr == "a":
|
|
72
|
+
return getattr(obj, attr)
|
|
73
|
+
else:
|
|
74
|
+
return getattr(obj, "b")[attr]
|
|
75
|
+
|
|
76
|
+
MyObjectList = type("MyObjectList", (ObjectList,), {})
|
|
77
|
+
MyObjectList._get_obj_attr = my_get_obj_attr
|
|
78
|
+
|
|
79
|
+
o1 = MyObjectList(
|
|
80
|
+
[
|
|
81
|
+
{"a": 1, "b": {"c1": 2}},
|
|
82
|
+
{"a": 2, "b": {"c1": 3}},
|
|
83
|
+
]
|
|
84
|
+
)
|
|
85
|
+
assert o1.get(c1=2) == o1[0]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_get_with_missing_key(null_config) -> None:
|
|
89
|
+
o1 = ObjectList([{"a": 1}, {"b": 2}])
|
|
90
|
+
assert o1.get(a=1) == {"a": 1}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_parameters_list_get_equivalence(null_config) -> None:
|
|
94
|
+
p_name = "p12334567898765432101"
|
|
95
|
+
hf.parameters.add_object(hf.Parameter(p_name))
|
|
96
|
+
assert p_name in hf.parameters.list_attrs()
|
|
97
|
+
assert (
|
|
98
|
+
getattr(hf.parameters, p_name)
|
|
99
|
+
== hf.parameters.get(p_name)
|
|
100
|
+
== hf.parameters.get_all(p_name)[0]
|
|
101
|
+
== hf.parameters.get(typ=p_name)
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_parameters_list_get_equivalence_non_existent(new_null_config) -> None:
|
|
106
|
+
# non-existent parameters should be created, unlike other ObjectList sub-classes,
|
|
107
|
+
# which raise
|
|
108
|
+
hf.reload_template_components()
|
|
109
|
+
p_name = "p12334567898765432101"
|
|
110
|
+
assert p_name not in hf.parameters.list_attrs()
|
|
111
|
+
assert (
|
|
112
|
+
getattr(hf.parameters, p_name)
|
|
113
|
+
== hf.parameters.get(p_name)
|
|
114
|
+
== hf.parameters.get_all(p_name)[0]
|
|
115
|
+
== hf.parameters.get(typ=p_name)
|
|
116
|
+
)
|