hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/__pyinstaller/hook-hpcflow.py +9 -6
- hpcflow/_version.py +1 -1
- hpcflow/app.py +1 -0
- hpcflow/data/scripts/bad_script.py +2 -0
- hpcflow/data/scripts/do_nothing.py +2 -0
- hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py +4 -0
- hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py +8 -0
- hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py +4 -0
- hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py +4 -0
- hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py +7 -0
- hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py +4 -0
- hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py +7 -0
- hpcflow/data/scripts/input_file_generator_basic.py +3 -0
- hpcflow/data/scripts/input_file_generator_basic_FAIL.py +3 -0
- hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py +8 -0
- hpcflow/data/scripts/main_script_test_direct_in.py +3 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py +7 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py +6 -0
- hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
- hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py +12 -0
- hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
- hpcflow/data/scripts/main_script_test_json_out_FAIL.py +3 -0
- hpcflow/data/scripts/main_script_test_shell_env_vars.py +12 -0
- hpcflow/data/scripts/main_script_test_std_out_std_err.py +6 -0
- hpcflow/data/scripts/output_file_parser_basic.py +3 -0
- hpcflow/data/scripts/output_file_parser_basic_FAIL.py +7 -0
- hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py +8 -0
- hpcflow/data/scripts/script_exit_test.py +5 -0
- hpcflow/data/template_components/environments.yaml +1 -1
- hpcflow/sdk/__init__.py +26 -15
- hpcflow/sdk/app.py +2192 -768
- hpcflow/sdk/cli.py +506 -296
- hpcflow/sdk/cli_common.py +105 -7
- hpcflow/sdk/config/__init__.py +1 -1
- hpcflow/sdk/config/callbacks.py +115 -43
- hpcflow/sdk/config/cli.py +126 -103
- hpcflow/sdk/config/config.py +674 -318
- hpcflow/sdk/config/config_file.py +131 -95
- hpcflow/sdk/config/errors.py +125 -84
- hpcflow/sdk/config/types.py +148 -0
- hpcflow/sdk/core/__init__.py +25 -1
- hpcflow/sdk/core/actions.py +1771 -1059
- hpcflow/sdk/core/app_aware.py +24 -0
- hpcflow/sdk/core/cache.py +139 -79
- hpcflow/sdk/core/command_files.py +263 -287
- hpcflow/sdk/core/commands.py +145 -112
- hpcflow/sdk/core/element.py +828 -535
- hpcflow/sdk/core/enums.py +192 -0
- hpcflow/sdk/core/environment.py +74 -93
- hpcflow/sdk/core/errors.py +455 -52
- hpcflow/sdk/core/execute.py +207 -0
- hpcflow/sdk/core/json_like.py +540 -272
- hpcflow/sdk/core/loop.py +751 -347
- hpcflow/sdk/core/loop_cache.py +164 -47
- hpcflow/sdk/core/object_list.py +370 -207
- hpcflow/sdk/core/parameters.py +1100 -627
- hpcflow/sdk/core/rule.py +59 -41
- hpcflow/sdk/core/run_dir_files.py +21 -37
- hpcflow/sdk/core/skip_reason.py +7 -0
- hpcflow/sdk/core/task.py +1649 -1339
- hpcflow/sdk/core/task_schema.py +308 -196
- hpcflow/sdk/core/test_utils.py +191 -114
- hpcflow/sdk/core/types.py +440 -0
- hpcflow/sdk/core/utils.py +485 -309
- hpcflow/sdk/core/validation.py +82 -9
- hpcflow/sdk/core/workflow.py +2544 -1178
- hpcflow/sdk/core/zarr_io.py +98 -137
- hpcflow/sdk/data/workflow_spec_schema.yaml +2 -0
- hpcflow/sdk/demo/cli.py +53 -33
- hpcflow/sdk/helper/cli.py +18 -15
- hpcflow/sdk/helper/helper.py +75 -63
- hpcflow/sdk/helper/watcher.py +61 -28
- hpcflow/sdk/log.py +122 -71
- hpcflow/sdk/persistence/__init__.py +8 -31
- hpcflow/sdk/persistence/base.py +1360 -606
- hpcflow/sdk/persistence/defaults.py +6 -0
- hpcflow/sdk/persistence/discovery.py +38 -0
- hpcflow/sdk/persistence/json.py +568 -188
- hpcflow/sdk/persistence/pending.py +382 -179
- hpcflow/sdk/persistence/store_resource.py +39 -23
- hpcflow/sdk/persistence/types.py +318 -0
- hpcflow/sdk/persistence/utils.py +14 -11
- hpcflow/sdk/persistence/zarr.py +1337 -433
- hpcflow/sdk/runtime.py +44 -41
- hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
- hpcflow/sdk/submission/jobscript.py +1651 -692
- hpcflow/sdk/submission/schedulers/__init__.py +167 -39
- hpcflow/sdk/submission/schedulers/direct.py +121 -81
- hpcflow/sdk/submission/schedulers/sge.py +170 -129
- hpcflow/sdk/submission/schedulers/slurm.py +291 -268
- hpcflow/sdk/submission/schedulers/utils.py +12 -2
- hpcflow/sdk/submission/shells/__init__.py +14 -15
- hpcflow/sdk/submission/shells/base.py +150 -29
- hpcflow/sdk/submission/shells/bash.py +283 -173
- hpcflow/sdk/submission/shells/os_version.py +31 -30
- hpcflow/sdk/submission/shells/powershell.py +228 -170
- hpcflow/sdk/submission/submission.py +1014 -335
- hpcflow/sdk/submission/types.py +140 -0
- hpcflow/sdk/typing.py +182 -12
- hpcflow/sdk/utils/arrays.py +71 -0
- hpcflow/sdk/utils/deferred_file.py +55 -0
- hpcflow/sdk/utils/hashing.py +16 -0
- hpcflow/sdk/utils/patches.py +12 -0
- hpcflow/sdk/utils/strings.py +33 -0
- hpcflow/tests/api/test_api.py +32 -0
- hpcflow/tests/conftest.py +27 -6
- hpcflow/tests/data/multi_path_sequences.yaml +29 -0
- hpcflow/tests/data/workflow_test_run_abort.yaml +34 -35
- hpcflow/tests/schedulers/sge/test_sge_submission.py +36 -0
- hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
- hpcflow/tests/scripts/test_input_file_generators.py +282 -0
- hpcflow/tests/scripts/test_main_scripts.py +866 -85
- hpcflow/tests/scripts/test_non_snippet_script.py +46 -0
- hpcflow/tests/scripts/test_ouput_file_parsers.py +353 -0
- hpcflow/tests/shells/wsl/test_wsl_submission.py +12 -4
- hpcflow/tests/unit/test_action.py +262 -75
- hpcflow/tests/unit/test_action_rule.py +9 -4
- hpcflow/tests/unit/test_app.py +33 -6
- hpcflow/tests/unit/test_cache.py +46 -0
- hpcflow/tests/unit/test_cli.py +134 -1
- hpcflow/tests/unit/test_command.py +71 -54
- hpcflow/tests/unit/test_config.py +142 -16
- hpcflow/tests/unit/test_config_file.py +21 -18
- hpcflow/tests/unit/test_element.py +58 -62
- hpcflow/tests/unit/test_element_iteration.py +50 -1
- hpcflow/tests/unit/test_element_set.py +29 -19
- hpcflow/tests/unit/test_group.py +4 -2
- hpcflow/tests/unit/test_input_source.py +116 -93
- hpcflow/tests/unit/test_input_value.py +29 -24
- hpcflow/tests/unit/test_jobscript_unit.py +757 -0
- hpcflow/tests/unit/test_json_like.py +44 -35
- hpcflow/tests/unit/test_loop.py +1396 -84
- hpcflow/tests/unit/test_meta_task.py +325 -0
- hpcflow/tests/unit/test_multi_path_sequences.py +229 -0
- hpcflow/tests/unit/test_object_list.py +17 -12
- hpcflow/tests/unit/test_parameter.py +29 -7
- hpcflow/tests/unit/test_persistence.py +237 -42
- hpcflow/tests/unit/test_resources.py +20 -18
- hpcflow/tests/unit/test_run.py +117 -6
- hpcflow/tests/unit/test_run_directories.py +29 -0
- hpcflow/tests/unit/test_runtime.py +2 -1
- hpcflow/tests/unit/test_schema_input.py +23 -15
- hpcflow/tests/unit/test_shell.py +23 -2
- hpcflow/tests/unit/test_slurm.py +8 -7
- hpcflow/tests/unit/test_submission.py +38 -89
- hpcflow/tests/unit/test_task.py +352 -247
- hpcflow/tests/unit/test_task_schema.py +33 -20
- hpcflow/tests/unit/test_utils.py +9 -11
- hpcflow/tests/unit/test_value_sequence.py +15 -12
- hpcflow/tests/unit/test_workflow.py +114 -83
- hpcflow/tests/unit/test_workflow_template.py +0 -1
- hpcflow/tests/unit/utils/test_arrays.py +40 -0
- hpcflow/tests/unit/utils/test_deferred_file_writer.py +34 -0
- hpcflow/tests/unit/utils/test_hashing.py +65 -0
- hpcflow/tests/unit/utils/test_patches.py +5 -0
- hpcflow/tests/unit/utils/test_redirect_std.py +50 -0
- hpcflow/tests/workflows/__init__.py +0 -0
- hpcflow/tests/workflows/test_directory_structure.py +31 -0
- hpcflow/tests/workflows/test_jobscript.py +334 -1
- hpcflow/tests/workflows/test_run_status.py +198 -0
- hpcflow/tests/workflows/test_skip_downstream.py +696 -0
- hpcflow/tests/workflows/test_submission.py +140 -0
- hpcflow/tests/workflows/test_workflows.py +160 -15
- hpcflow/tests/workflows/test_zip.py +18 -0
- hpcflow/viz_demo.ipynb +6587 -3
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/METADATA +8 -4
- hpcflow_new2-0.2.0a199.dist-info/RECORD +221 -0
- hpcflow/sdk/core/parallel.py +0 -21
- hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/entry_points.txt +0 -0
@@ -1,58 +1,92 @@
|
|
1
1
|
"""
|
2
|
-
Shell models based on the Bourne-Again Shell.
|
2
|
+
Shell models based on the GNU Bourne-Again Shell.
|
3
3
|
"""
|
4
4
|
|
5
|
+
from __future__ import annotations
|
5
6
|
from pathlib import Path
|
6
7
|
import subprocess
|
7
8
|
from textwrap import dedent, indent
|
8
|
-
from typing import
|
9
|
+
from typing import TYPE_CHECKING
|
10
|
+
from typing_extensions import override
|
11
|
+
from hpcflow.sdk.typing import hydrate
|
9
12
|
from hpcflow.sdk.core import ABORT_EXIT_CODE
|
10
|
-
from hpcflow.sdk.submission.shells import Shell
|
13
|
+
from hpcflow.sdk.submission.shells.base import Shell
|
11
14
|
from hpcflow.sdk.submission.shells.os_version import (
|
12
15
|
get_OS_info_POSIX,
|
13
16
|
get_OS_info_windows,
|
14
17
|
)
|
15
18
|
|
19
|
+
if TYPE_CHECKING:
|
20
|
+
from collections.abc import Mapping
|
21
|
+
from typing import Any, ClassVar
|
22
|
+
from .base import VersionInfo, JobscriptHeaderArgs
|
16
23
|
|
24
|
+
|
25
|
+
@hydrate
|
17
26
|
class Bash(Shell):
|
18
27
|
"""
|
19
28
|
Class to represent using bash on a POSIX OS to generate and submit a jobscript.
|
20
29
|
"""
|
21
30
|
|
22
31
|
#: Default for executable name.
|
23
|
-
DEFAULT_EXE = "/bin/bash"
|
32
|
+
DEFAULT_EXE: ClassVar[str] = "/bin/bash"
|
24
33
|
|
25
34
|
#: File extension for jobscripts.
|
26
|
-
JS_EXT = ".sh"
|
35
|
+
JS_EXT: ClassVar[str] = ".sh"
|
27
36
|
#: Basic indent.
|
28
|
-
JS_INDENT = " "
|
37
|
+
JS_INDENT: ClassVar[str] = " "
|
29
38
|
#: Indent for environment setup.
|
30
|
-
JS_ENV_SETUP_INDENT = 2 * JS_INDENT
|
39
|
+
JS_ENV_SETUP_INDENT: ClassVar[str] = 2 * JS_INDENT
|
31
40
|
#: Template for the jobscript shebang line.
|
32
|
-
JS_SHEBANG = """#!{shebang_executable} {shebang_args}"""
|
33
|
-
#: Template for the
|
34
|
-
|
41
|
+
JS_SHEBANG: ClassVar[str] = """#!{shebang_executable} {shebang_args}"""
|
42
|
+
#: Template for the jobscript functions file.
|
43
|
+
JS_FUNCS: ClassVar[str] = dedent(
|
35
44
|
"""\
|
36
45
|
{workflow_app_alias} () {{
|
37
46
|
(
|
38
47
|
{env_setup}{app_invoc}\\
|
39
|
-
--with-config log_file_path "
|
48
|
+
--with-config log_file_path "${app_caps}_LOG_PATH"\\
|
40
49
|
--config-dir "{config_dir}"\\
|
41
50
|
--config-key "{config_invoc_key}"\\
|
42
51
|
"$@"
|
43
52
|
)
|
44
53
|
}}
|
45
|
-
|
54
|
+
"""
|
55
|
+
)
|
56
|
+
#: Template for the common part of the jobscript header.
|
57
|
+
JS_HEADER: ClassVar[str] = dedent(
|
58
|
+
"""\
|
46
59
|
WK_PATH=`pwd`
|
47
60
|
WK_PATH_ARG="$WK_PATH"
|
48
61
|
SUB_IDX={sub_idx}
|
49
62
|
JS_IDX={js_idx}
|
50
|
-
|
51
|
-
|
63
|
+
APP_CAPS={app_caps}
|
64
|
+
|
65
|
+
SUB_DIR="$WK_PATH/artifacts/submissions/${{SUB_IDX}}"
|
66
|
+
JS_FUNCS_PATH="$SUB_DIR/{jobscript_functions_dir}/{jobscript_functions_name}"
|
67
|
+
. "$JS_FUNCS_PATH"
|
68
|
+
|
69
|
+
EAR_ID_FILE="$WK_PATH/artifacts/submissions/${{SUB_IDX}}/{run_IDs_file_dir}/{run_IDs_file_name}"
|
70
|
+
SUB_TMP_DIR="$SUB_DIR/{tmp_dir_name}"
|
71
|
+
SUB_LOG_DIR="$SUB_DIR/{log_dir_name}"
|
72
|
+
SUB_STD_DIR="$SUB_DIR/{app_std_dir_name}"
|
73
|
+
SUB_SCRIPTS_DIR="$SUB_DIR/{scripts_dir_name}"
|
74
|
+
|
75
|
+
export {app_caps}_WK_PATH=$WK_PATH
|
76
|
+
export {app_caps}_WK_PATH_ARG=$WK_PATH_ARG
|
77
|
+
export {app_caps}_SUB_IDX={sub_idx}
|
78
|
+
export {app_caps}_SUB_SCRIPTS_DIR=$SUB_SCRIPTS_DIR
|
79
|
+
export {app_caps}_SUB_TMP_DIR=$SUB_TMP_DIR
|
80
|
+
export {app_caps}_SUB_LOG_DIR=$SUB_LOG_DIR
|
81
|
+
export {app_caps}_SUB_STD_DIR=$SUB_STD_DIR
|
82
|
+
export {app_caps}_LOG_PATH="$SUB_LOG_DIR/js_${{JS_IDX}}.log"
|
83
|
+
export {app_caps}_JS_FUNCS_PATH=$JS_FUNCS_PATH
|
84
|
+
export {app_caps}_JS_IDX={js_idx}
|
85
|
+
export {app_caps}_RUN_ID_FILE=$EAR_ID_FILE
|
52
86
|
"""
|
53
87
|
)
|
54
88
|
#: Template for the jobscript header when scheduled.
|
55
|
-
JS_SCHEDULER_HEADER = dedent(
|
89
|
+
JS_SCHEDULER_HEADER: ClassVar[str] = dedent(
|
56
90
|
"""\
|
57
91
|
{shebang}
|
58
92
|
|
@@ -61,96 +95,136 @@ class Bash(Shell):
|
|
61
95
|
"""
|
62
96
|
)
|
63
97
|
#: Template for the jobscript header when directly executed.
|
64
|
-
JS_DIRECT_HEADER = dedent(
|
98
|
+
JS_DIRECT_HEADER: ClassVar[str] = dedent(
|
65
99
|
"""\
|
66
100
|
{shebang}
|
67
|
-
|
68
101
|
{header}
|
69
102
|
{wait_command}
|
70
103
|
"""
|
71
104
|
)
|
72
|
-
#: Template for the
|
73
|
-
|
105
|
+
#: Template for enabling writing of the app log.
|
106
|
+
JS_RUN_LOG_PATH_ENABLE: ClassVar[str] = '"$SUB_LOG_DIR/{run_log_file_name}"'
|
107
|
+
#: Template for disabling writing of the app log.
|
108
|
+
JS_RUN_LOG_PATH_DISABLE: ClassVar[str] = '" "'
|
109
|
+
#: Template for the run execution command.
|
110
|
+
JS_RUN_CMD: ClassVar[str] = (
|
111
|
+
'{workflow_app_alias} internal workflow "$WK_PATH_ARG" execute-run '
|
112
|
+
"$SUB_IDX $JS_IDX $block_idx $block_act_idx $EAR_ID\n"
|
113
|
+
)
|
114
|
+
#: Template for the execution command for multiple combined runs.
|
115
|
+
JS_RUN_CMD_COMBINED: ClassVar[str] = (
|
116
|
+
'{workflow_app_alias} internal workflow "$WK_PATH_ARG" execute-combined-runs '
|
117
|
+
"$SUB_IDX $JS_IDX\n"
|
118
|
+
)
|
119
|
+
#: Template for setting up run environment variables and executing the run.
|
120
|
+
JS_RUN: ClassVar[str] = dedent(
|
74
121
|
"""\
|
122
|
+
EAR_ID="$(cut -d'{EAR_files_delimiter}' -f $(($block_act_idx + 1)) <<< $elem_EAR_IDs)"
|
123
|
+
if [ "$EAR_ID" = "-1" ]; then
|
124
|
+
continue
|
125
|
+
fi
|
126
|
+
|
127
|
+
export {app_caps}_RUN_ID=$EAR_ID
|
128
|
+
export {app_caps}_RUN_LOG_PATH={run_log_enable_disable}
|
129
|
+
export {app_caps}_LOG_PATH="${app_caps}_RUN_LOG_PATH"
|
130
|
+
export {app_caps}_RUN_STD_PATH="$SUB_STD_DIR/${app_caps}_RUN_ID.txt"
|
131
|
+
export {app_caps}_BLOCK_ACT_IDX=$block_act_idx
|
132
|
+
|
133
|
+
cd "$SUB_TMP_DIR"
|
134
|
+
|
135
|
+
{run_cmd}
|
136
|
+
"""
|
137
|
+
)
|
138
|
+
#: Template for the action-run processing loop in a jobscript.
|
139
|
+
JS_ACT_MULTI: ClassVar[str] = dedent(
|
140
|
+
"""\
|
141
|
+
for ((block_act_idx=0;block_act_idx<{num_actions};block_act_idx++))
|
142
|
+
do
|
143
|
+
{run_block}
|
144
|
+
done
|
145
|
+
"""
|
146
|
+
)
|
147
|
+
#: Template for the single-action-run execution in a jobscript.
|
148
|
+
JS_ACT_SINGLE: ClassVar[str] = dedent(
|
149
|
+
"""\
|
150
|
+
block_act_idx=0
|
151
|
+
{run_block}
|
152
|
+
"""
|
153
|
+
)
|
154
|
+
#: Template for setting up environment variables and running one or more action-runs.
|
155
|
+
JS_MAIN: ClassVar[str] = dedent(
|
156
|
+
"""\
|
157
|
+
block_elem_idx=$(( $JS_elem_idx - {block_start_elem_idx} ))
|
75
158
|
elem_EAR_IDs=`sed "$((${{JS_elem_idx}} + 1))q;d" "$EAR_ID_FILE"`
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
if [ "$skip" = "1" ]; then
|
96
|
-
continue
|
97
|
-
fi
|
98
|
-
|
99
|
-
{workflow_app_alias} internal workflow "$WK_PATH_ARG" write-commands $SUB_IDX $JS_IDX $JS_act_idx $EAR_ID >> "$app_stream_file" 2>&1
|
100
|
-
exc_wc=$?
|
101
|
-
|
102
|
-
{workflow_app_alias} internal workflow "$WK_PATH_ARG" set-ear-start $EAR_ID >> "$app_stream_file" 2>&1
|
103
|
-
exc_se=$?
|
104
|
-
|
105
|
-
if [ $exc_wc -eq 0 ] && [ $exc_se -eq 0 ]; then
|
106
|
-
. {commands_file_name}
|
107
|
-
exit_code=$?
|
108
|
-
else
|
109
|
-
exit_code=$([ $exc_wc -ne 0 ] && echo "$exc_wc" || echo "$exc_se")
|
110
|
-
fi
|
111
|
-
|
112
|
-
else
|
113
|
-
exit_code=$exc_sk
|
114
|
-
fi
|
115
|
-
|
116
|
-
{workflow_app_alias} internal workflow "$WK_PATH_ARG" set-ear-end $JS_IDX $JS_act_idx $EAR_ID "--" "$exit_code" >> "$app_stream_file" 2>&1
|
117
|
-
|
118
|
-
done
|
159
|
+
export {app_caps}_JS_ELEM_IDX=$JS_elem_idx
|
160
|
+
export {app_caps}_BLOCK_ELEM_IDX=$block_elem_idx
|
161
|
+
|
162
|
+
{action}
|
163
|
+
"""
|
164
|
+
)
|
165
|
+
#: Template for a jobscript-block header.
|
166
|
+
JS_BLOCK_HEADER: ClassVar[str] = dedent( # for single-block jobscripts only
|
167
|
+
"""\
|
168
|
+
block_idx=0
|
169
|
+
export {app_caps}_BLOCK_IDX=0
|
170
|
+
"""
|
171
|
+
)
|
172
|
+
#: Template for single-element execution.
|
173
|
+
JS_ELEMENT_SINGLE: ClassVar[str] = dedent(
|
174
|
+
"""\
|
175
|
+
JS_elem_idx={block_start_elem_idx}
|
176
|
+
{main}
|
119
177
|
"""
|
120
178
|
)
|
121
179
|
#: Template for the element processing loop in a jobscript.
|
122
|
-
|
180
|
+
JS_ELEMENT_MULTI_LOOP: ClassVar[str] = dedent(
|
123
181
|
"""\
|
124
|
-
for ((JS_elem_idx=
|
182
|
+
for ((JS_elem_idx={block_start_elem_idx};JS_elem_idx<$(({block_start_elem_idx} + {num_elements}));JS_elem_idx++))
|
125
183
|
do
|
126
184
|
{main}
|
127
185
|
done
|
128
|
-
cd "$WK_PATH"
|
129
186
|
"""
|
130
187
|
)
|
131
188
|
#: Template for the array handling code in a jobscript.
|
132
|
-
|
189
|
+
JS_ELEMENT_MULTI_ARRAY: ClassVar[str] = dedent(
|
133
190
|
"""\
|
134
191
|
JS_elem_idx=$(({scheduler_array_item_var} - 1))
|
135
192
|
{main}
|
136
|
-
cd "$WK_PATH"
|
137
193
|
"""
|
138
194
|
)
|
139
|
-
|
140
|
-
|
141
|
-
|
195
|
+
#: Template for the jobscript block loop in a jobscript.
|
196
|
+
JS_BLOCK_LOOP: ClassVar[str] = dedent(
|
197
|
+
"""\
|
198
|
+
num_elements={num_elements}
|
199
|
+
num_actions={num_actions}
|
200
|
+
block_start_elem_idx=0
|
201
|
+
for ((block_idx=0;block_idx<{num_blocks};block_idx++))
|
202
|
+
do
|
203
|
+
export {app_caps}_BLOCK_IDX=$block_idx
|
204
|
+
{element_loop}
|
205
|
+
block_start_elem_idx=$(($block_start_elem_idx + ${{num_elements[$block_idx]}}))
|
206
|
+
done
|
207
|
+
"""
|
208
|
+
)
|
209
|
+
#: Template for the jobscript footer.
|
210
|
+
JS_FOOTER: ClassVar[str] = dedent(
|
211
|
+
"""\
|
212
|
+
cd $WK_PATH
|
213
|
+
"""
|
214
|
+
)
|
142
215
|
|
143
216
|
@property
|
144
|
-
def linux_release_file(self):
|
217
|
+
def linux_release_file(self) -> str:
|
145
218
|
"""
|
146
219
|
The name of the file describing the Linux version.
|
147
220
|
"""
|
148
221
|
return self.os_args["linux_release_file"]
|
149
222
|
|
150
|
-
def _get_OS_info_POSIX(self):
|
223
|
+
def _get_OS_info_POSIX(self) -> Mapping[str, str]:
|
151
224
|
return get_OS_info_POSIX(linux_release_file=self.linux_release_file)
|
152
225
|
|
153
|
-
|
226
|
+
@override
|
227
|
+
def get_version_info(self, exclude_os: bool = False) -> VersionInfo:
|
154
228
|
"""Get bash version information.
|
155
229
|
|
156
230
|
Parameters
|
@@ -171,37 +245,74 @@ class Bash(Shell):
|
|
171
245
|
else:
|
172
246
|
raise RuntimeError("Failed to parse bash version information.")
|
173
247
|
|
174
|
-
|
248
|
+
return {
|
175
249
|
"shell_name": "bash",
|
176
250
|
"shell_executable": self.executable,
|
177
251
|
"shell_version": bash_version,
|
252
|
+
**({} if exclude_os else self._get_OS_info_POSIX()),
|
178
253
|
}
|
179
254
|
|
180
|
-
if not exclude_os:
|
181
|
-
out.update(**self._get_OS_info_POSIX())
|
182
|
-
|
183
|
-
return out
|
184
|
-
|
185
255
|
@staticmethod
|
186
|
-
def process_app_invoc_executable(app_invoc_exe):
|
256
|
+
def process_app_invoc_executable(app_invoc_exe: str) -> str:
|
187
257
|
# escape spaces with a back slash:
|
188
|
-
|
189
|
-
|
258
|
+
return app_invoc_exe.replace(" ", r"\ ")
|
259
|
+
|
260
|
+
@override
|
261
|
+
def format_env_var_get(self, var: str) -> str:
|
262
|
+
"""
|
263
|
+
Format retrieval of a shell environment variable.
|
264
|
+
"""
|
265
|
+
return f"${var}"
|
266
|
+
|
267
|
+
@override
|
268
|
+
def format_array(self, lst: list) -> str:
|
269
|
+
"""
|
270
|
+
Format construction of a shell array.
|
271
|
+
"""
|
272
|
+
return "(" + " ".join(str(i) for i in lst) + ")"
|
190
273
|
|
191
|
-
|
274
|
+
@override
|
275
|
+
def format_array_get_item(self, arr_name: str, index: int | str) -> str:
|
276
|
+
"""
|
277
|
+
Format retrieval of a shell array item at a specified index.
|
278
|
+
"""
|
279
|
+
return f"${{{arr_name}[{index}]}}"
|
280
|
+
|
281
|
+
@override
|
282
|
+
def format_stream_assignment(self, shell_var_name: str, command: str) -> str:
|
192
283
|
"""
|
193
284
|
Produce code to assign the output of the command to a shell variable.
|
194
285
|
"""
|
195
286
|
return f"{shell_var_name}=`{command}`"
|
196
287
|
|
288
|
+
@override
|
289
|
+
def format_source_functions_file(self, app_name: str, commands: str) -> str:
|
290
|
+
"""
|
291
|
+
Format sourcing (i.e. invocation) of the jobscript functions file.
|
292
|
+
"""
|
293
|
+
return dedent(
|
294
|
+
"""\
|
295
|
+
. "${app_caps}_JS_FUNCS_PATH"
|
296
|
+
|
297
|
+
"""
|
298
|
+
).format(app_caps=app_name.upper())
|
299
|
+
|
300
|
+
@override
|
301
|
+
def format_commands_file(self, app_name: str, commands: str) -> str:
|
302
|
+
"""
|
303
|
+
Format the commands file.
|
304
|
+
"""
|
305
|
+
return self.format_source_functions_file(app_name, commands) + commands
|
306
|
+
|
307
|
+
@override
|
197
308
|
def format_save_parameter(
|
198
309
|
self,
|
199
310
|
workflow_app_alias: str,
|
200
311
|
param_name: str,
|
201
312
|
shell_var_name: str,
|
202
|
-
EAR_ID: int,
|
203
313
|
cmd_idx: int,
|
204
314
|
stderr: bool,
|
315
|
+
app_name: str,
|
205
316
|
):
|
206
317
|
"""
|
207
318
|
Produce code to save a parameter's value into the workflow persistent store.
|
@@ -209,72 +320,14 @@ class Bash(Shell):
|
|
209
320
|
# TODO: quote shell_var_name as well? e.g. if it's a white-space delimited list?
|
210
321
|
# and test.
|
211
322
|
stderr_str = " --stderr" if stderr else ""
|
323
|
+
app_caps = app_name.upper()
|
212
324
|
return (
|
213
|
-
f
|
214
|
-
f'internal workflow "$
|
215
|
-
f"{param_name} ${shell_var_name} {
|
216
|
-
f'>> "$app_stream_file" 2>&1'
|
325
|
+
f'{workflow_app_alias} --std-stream "${app_caps}_RUN_STD_PATH" '
|
326
|
+
f'internal workflow "${app_caps}_WK_PATH_ARG" save-parameter {stderr_str}'
|
327
|
+
f'"--" {param_name} ${shell_var_name} ${app_caps}_RUN_ID {cmd_idx}'
|
217
328
|
f"\n"
|
218
329
|
)
|
219
330
|
|
220
|
-
def format_loop_check(self, workflow_app_alias: str, loop_name: str, run_ID: int):
|
221
|
-
"""
|
222
|
-
Produce code to check the looping status of part of a workflow.
|
223
|
-
"""
|
224
|
-
return (
|
225
|
-
f"{workflow_app_alias} "
|
226
|
-
f'internal workflow "$WK_PATH_ARG" check-loop '
|
227
|
-
f"{loop_name} {run_ID} "
|
228
|
-
f'>> "$app_stream_file" 2>&1'
|
229
|
-
f"\n"
|
230
|
-
)
|
231
|
-
|
232
|
-
def wrap_in_subshell(self, commands: str, abortable: bool) -> str:
|
233
|
-
"""Format commands to run within a subshell.
|
234
|
-
|
235
|
-
This assumes commands ends in a newline.
|
236
|
-
|
237
|
-
"""
|
238
|
-
commands = indent(commands, self.JS_INDENT)
|
239
|
-
if abortable:
|
240
|
-
# run commands in the background, and poll a file to check for abort requests:
|
241
|
-
return dedent(
|
242
|
-
"""\
|
243
|
-
(
|
244
|
-
{commands}) &
|
245
|
-
|
246
|
-
pid=$!
|
247
|
-
abort_file=$WK_PATH/artifacts/submissions/$SUB_IDX/abort_EARs.txt
|
248
|
-
while true
|
249
|
-
do
|
250
|
-
is_abort=`sed "$(($EAR_ID + 1))q;d" $abort_file`
|
251
|
-
ps -p $pid > /dev/null
|
252
|
-
if [ $? == 1 ]; then
|
253
|
-
wait $pid
|
254
|
-
exitcode=$?
|
255
|
-
break
|
256
|
-
elif [ "$is_abort" = "1" ]; then
|
257
|
-
echo "Abort instruction received; stopping commands..." >> "$app_stream_file"
|
258
|
-
kill $pid
|
259
|
-
wait $pid 2>/dev/null
|
260
|
-
exitcode={abort_exit_code}
|
261
|
-
break
|
262
|
-
else
|
263
|
-
sleep 1 # TODO: TEMP: increase for production
|
264
|
-
fi
|
265
|
-
done
|
266
|
-
return $exitcode
|
267
|
-
"""
|
268
|
-
).format(commands=commands, abort_exit_code=ABORT_EXIT_CODE)
|
269
|
-
else:
|
270
|
-
# run commands in "foreground":
|
271
|
-
return dedent(
|
272
|
-
"""\
|
273
|
-
(
|
274
|
-
{commands})
|
275
|
-
"""
|
276
|
-
).format(commands=commands)
|
277
|
-
|
278
331
|
|
279
332
|
class WSLBash(Bash):
|
280
333
|
"""
|
@@ -282,38 +335,87 @@ class WSLBash(Bash):
|
|
282
335
|
"""
|
283
336
|
|
284
337
|
#: Default name of the WSL interface executable.
|
285
|
-
DEFAULT_WSL_EXE = "wsl"
|
338
|
+
DEFAULT_WSL_EXE: ClassVar[str] = "wsl"
|
286
339
|
|
340
|
+
#: Template for the jobscript functions file.
|
341
|
+
JS_FUNCS: ClassVar[str] = dedent(
|
342
|
+
"""\
|
343
|
+
{{workflow_app_alias}} () {{{{
|
344
|
+
(
|
345
|
+
{log_path_block}
|
346
|
+
{{env_setup}}{{app_invoc}}\\
|
347
|
+
--with-config log_file_path "$LOG_FILE_PATH"\\
|
348
|
+
--config-dir "{{config_dir}}"\\
|
349
|
+
--config-key "{{config_invoc_key}}"\\
|
350
|
+
"$@"
|
351
|
+
)
|
352
|
+
}}}}
|
353
|
+
"""
|
354
|
+
).format(
|
355
|
+
log_path_block=indent(
|
356
|
+
dedent(
|
357
|
+
"""\
|
358
|
+
if [ -z "${app_caps}_LOG_PATH" ] || [ "${app_caps}_LOG_PATH" = " " ]; then
|
359
|
+
LOG_FILE_PATH=" "
|
360
|
+
else
|
361
|
+
LOG_FILE_PATH="$(wslpath -m ${app_caps}_LOG_PATH)"
|
362
|
+
fi
|
363
|
+
"""
|
364
|
+
),
|
365
|
+
prefix=Bash.JS_ENV_SETUP_INDENT,
|
366
|
+
)
|
367
|
+
)
|
287
368
|
#: Template for the common part of the jobscript header.
|
288
|
-
JS_HEADER = Bash.JS_HEADER.replace(
|
369
|
+
JS_HEADER: ClassVar[str] = Bash.JS_HEADER.replace(
|
289
370
|
'WK_PATH_ARG="$WK_PATH"',
|
290
371
|
'WK_PATH_ARG=`wslpath -m "$WK_PATH"`',
|
291
|
-
)
|
292
|
-
|
293
|
-
|
372
|
+
)
|
373
|
+
#: Template for the run execution command.
|
374
|
+
JS_RUN_CMD: ClassVar[str] = (
|
375
|
+
dedent(
|
376
|
+
"""\
|
377
|
+
WSLENV=$WSLENV:${{APP_CAPS}}_WK_PATH
|
378
|
+
WSLENV=$WSLENV:${{APP_CAPS}}_WK_PATH_ARG
|
379
|
+
WSLENV=$WSLENV:${{APP_CAPS}}_JS_FUNCS_PATH
|
380
|
+
WSLENV=$WSLENV:${{APP_CAPS}}_STD_STREAM_FILE
|
381
|
+
WSLENV=$WSLENV:${{APP_CAPS}}_SUB_IDX
|
382
|
+
WSLENV=$WSLENV:${{APP_CAPS}}_JS_IDX
|
383
|
+
WSLENV=$WSLENV:${{APP_CAPS}}_RUN_ID
|
384
|
+
WSLENV=$WSLENV:${{APP_CAPS}}_BLOCK_ACT_IDX
|
385
|
+
WSLENV=$WSLENV:${{APP_CAPS}}_JS_ELEM_IDX
|
386
|
+
WSLENV=$WSLENV:${{APP_CAPS}}_BLOCK_ELEM_IDX
|
387
|
+
WSLENV=$WSLENV:${{APP_CAPS}}_BLOCK_IDX
|
388
|
+
WSLENV=$WSLENV:${{APP_CAPS}}_LOG_PATH/p
|
389
|
+
|
390
|
+
"""
|
391
|
+
)
|
392
|
+
+ Bash.JS_RUN_CMD
|
294
393
|
)
|
295
394
|
|
296
395
|
def __init__(
|
297
396
|
self,
|
298
|
-
WSL_executable:
|
299
|
-
WSL_distribution:
|
300
|
-
WSL_user:
|
397
|
+
WSL_executable: str | None = None,
|
398
|
+
WSL_distribution: str | None = None,
|
399
|
+
WSL_user: str | None = None,
|
301
400
|
*args,
|
302
401
|
**kwargs,
|
303
402
|
):
|
403
|
+
#: The WSL executable wrapper.
|
304
404
|
self.WSL_executable = WSL_executable or self.DEFAULT_WSL_EXE
|
405
|
+
#: The WSL distribution to use, if any.
|
305
406
|
self.WSL_distribution = WSL_distribution
|
407
|
+
#: The WSL user to use, if any.
|
306
408
|
self.WSL_user = WSL_user
|
307
409
|
super().__init__(*args, **kwargs)
|
308
410
|
|
309
|
-
def __eq__(self, other) -> bool:
|
411
|
+
def __eq__(self, other: Any) -> bool:
|
310
412
|
return super().__eq__(other) and (
|
311
413
|
self.WSL_executable == other.WSL_executable
|
312
414
|
and self.WSL_distribution == other.WSL_distribution
|
313
415
|
and self.WSL_user == other.WSL_user
|
314
416
|
)
|
315
417
|
|
316
|
-
def _get_WSL_command(self):
|
418
|
+
def _get_WSL_command(self) -> list[str]:
|
317
419
|
out = [self.WSL_executable]
|
318
420
|
if self.WSL_distribution:
|
319
421
|
out += ["--distribution", self.WSL_distribution]
|
@@ -322,14 +424,14 @@ class WSLBash(Bash):
|
|
322
424
|
return out
|
323
425
|
|
324
426
|
@property
|
325
|
-
def executable(self) ->
|
427
|
+
def executable(self) -> list[str]:
|
326
428
|
return self._get_WSL_command() + super().executable
|
327
429
|
|
328
430
|
@property
|
329
|
-
def shebang_executable(self) ->
|
431
|
+
def shebang_executable(self) -> list[str]:
|
330
432
|
return super().executable
|
331
433
|
|
332
|
-
def _get_OS_info_POSIX(self):
|
434
|
+
def _get_OS_info_POSIX(self) -> Mapping[str, str]:
|
333
435
|
return get_OS_info_POSIX(
|
334
436
|
WSL_executable=self._get_WSL_command(),
|
335
437
|
use_py=False,
|
@@ -337,27 +439,29 @@ class WSLBash(Bash):
|
|
337
439
|
)
|
338
440
|
|
339
441
|
@staticmethod
|
340
|
-
def _convert_to_wsl_path(win_path:
|
442
|
+
def _convert_to_wsl_path(win_path: str | Path) -> str:
|
341
443
|
win_path = Path(win_path)
|
342
444
|
parts = list(win_path.parts)
|
343
445
|
parts[0] = f"/mnt/{win_path.drive.lower().rstrip(':')}"
|
344
|
-
|
345
|
-
return wsl_path
|
446
|
+
return "/".join(parts)
|
346
447
|
|
347
|
-
def process_JS_header_args(
|
448
|
+
def process_JS_header_args(
|
449
|
+
self, header_args: JobscriptHeaderArgs
|
450
|
+
) -> JobscriptHeaderArgs:
|
348
451
|
# convert executable windows paths to posix style as expected by WSL:
|
349
|
-
header_args["app_invoc"]
|
350
|
-
|
351
|
-
|
452
|
+
ai = header_args["app_invoc"]
|
453
|
+
if isinstance(ai, list):
|
454
|
+
ai[0] = self._convert_to_wsl_path(ai[0])
|
352
455
|
return super().process_JS_header_args(header_args)
|
353
456
|
|
354
457
|
def prepare_JS_path(self, js_path: Path) -> str:
|
355
458
|
return self._convert_to_wsl_path(js_path)
|
356
459
|
|
357
|
-
def prepare_element_run_dirs(self, run_dirs:
|
358
|
-
return [[
|
460
|
+
def prepare_element_run_dirs(self, run_dirs: list[list[Path]]) -> list[list[str]]:
|
461
|
+
return [[str(path).replace("\\", "/") for path in i] for i in run_dirs]
|
359
462
|
|
360
|
-
|
463
|
+
@override
|
464
|
+
def get_version_info(self, exclude_os: bool = False) -> VersionInfo:
|
361
465
|
"""Get WSL and bash version information.
|
362
466
|
|
363
467
|
Parameters
|
@@ -368,12 +472,14 @@ class WSLBash(Bash):
|
|
368
472
|
"""
|
369
473
|
vers_info = super().get_version_info(exclude_os=exclude_os)
|
370
474
|
|
371
|
-
vers_info["shell_name"] =
|
475
|
+
vers_info["shell_name"] = f"wsl+{vers_info['shell_name']}".lower()
|
372
476
|
vers_info["WSL_executable"] = self.WSL_executable
|
373
|
-
|
374
|
-
|
477
|
+
if self.WSL_distribution:
|
478
|
+
vers_info["WSL_distribution"] = self.WSL_distribution
|
479
|
+
if self.WSL_user:
|
480
|
+
vers_info["WSL_user"] = self.WSL_user
|
375
481
|
|
376
|
-
for key in
|
482
|
+
for key in tuple(vers_info):
|
377
483
|
if key.startswith("OS_"):
|
378
484
|
vers_info[f"WSL_{key}"] = vers_info.pop(key)
|
379
485
|
|
@@ -381,3 +487,7 @@ class WSLBash(Bash):
|
|
381
487
|
vers_info.update(**get_OS_info_windows())
|
382
488
|
|
383
489
|
return vers_info
|
490
|
+
|
491
|
+
def get_command_file_launch_command(self, cmd_file_path: str) -> list[str]:
|
492
|
+
"""Get the command for launching the commands file for a given run."""
|
493
|
+
return self.executable + [self._convert_to_wsl_path(cmd_file_path)]
|