hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. hpcflow/__pyinstaller/hook-hpcflow.py +9 -6
  2. hpcflow/_version.py +1 -1
  3. hpcflow/app.py +1 -0
  4. hpcflow/data/scripts/bad_script.py +2 -0
  5. hpcflow/data/scripts/do_nothing.py +2 -0
  6. hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py +4 -0
  7. hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py +8 -0
  8. hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py +4 -0
  9. hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py +4 -0
  10. hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py +7 -0
  11. hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py +4 -0
  12. hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py +7 -0
  13. hpcflow/data/scripts/input_file_generator_basic.py +3 -0
  14. hpcflow/data/scripts/input_file_generator_basic_FAIL.py +3 -0
  15. hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py +8 -0
  16. hpcflow/data/scripts/main_script_test_direct_in.py +3 -0
  17. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py +6 -0
  18. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py +6 -0
  19. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py +7 -0
  20. hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py +6 -0
  21. hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py +6 -0
  22. hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py +6 -0
  23. hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
  24. hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py +12 -0
  25. hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
  26. hpcflow/data/scripts/main_script_test_json_out_FAIL.py +3 -0
  27. hpcflow/data/scripts/main_script_test_shell_env_vars.py +12 -0
  28. hpcflow/data/scripts/main_script_test_std_out_std_err.py +6 -0
  29. hpcflow/data/scripts/output_file_parser_basic.py +3 -0
  30. hpcflow/data/scripts/output_file_parser_basic_FAIL.py +7 -0
  31. hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py +8 -0
  32. hpcflow/data/scripts/script_exit_test.py +5 -0
  33. hpcflow/data/template_components/environments.yaml +1 -1
  34. hpcflow/sdk/__init__.py +26 -15
  35. hpcflow/sdk/app.py +2192 -768
  36. hpcflow/sdk/cli.py +506 -296
  37. hpcflow/sdk/cli_common.py +105 -7
  38. hpcflow/sdk/config/__init__.py +1 -1
  39. hpcflow/sdk/config/callbacks.py +115 -43
  40. hpcflow/sdk/config/cli.py +126 -103
  41. hpcflow/sdk/config/config.py +674 -318
  42. hpcflow/sdk/config/config_file.py +131 -95
  43. hpcflow/sdk/config/errors.py +125 -84
  44. hpcflow/sdk/config/types.py +148 -0
  45. hpcflow/sdk/core/__init__.py +25 -1
  46. hpcflow/sdk/core/actions.py +1771 -1059
  47. hpcflow/sdk/core/app_aware.py +24 -0
  48. hpcflow/sdk/core/cache.py +139 -79
  49. hpcflow/sdk/core/command_files.py +263 -287
  50. hpcflow/sdk/core/commands.py +145 -112
  51. hpcflow/sdk/core/element.py +828 -535
  52. hpcflow/sdk/core/enums.py +192 -0
  53. hpcflow/sdk/core/environment.py +74 -93
  54. hpcflow/sdk/core/errors.py +455 -52
  55. hpcflow/sdk/core/execute.py +207 -0
  56. hpcflow/sdk/core/json_like.py +540 -272
  57. hpcflow/sdk/core/loop.py +751 -347
  58. hpcflow/sdk/core/loop_cache.py +164 -47
  59. hpcflow/sdk/core/object_list.py +370 -207
  60. hpcflow/sdk/core/parameters.py +1100 -627
  61. hpcflow/sdk/core/rule.py +59 -41
  62. hpcflow/sdk/core/run_dir_files.py +21 -37
  63. hpcflow/sdk/core/skip_reason.py +7 -0
  64. hpcflow/sdk/core/task.py +1649 -1339
  65. hpcflow/sdk/core/task_schema.py +308 -196
  66. hpcflow/sdk/core/test_utils.py +191 -114
  67. hpcflow/sdk/core/types.py +440 -0
  68. hpcflow/sdk/core/utils.py +485 -309
  69. hpcflow/sdk/core/validation.py +82 -9
  70. hpcflow/sdk/core/workflow.py +2544 -1178
  71. hpcflow/sdk/core/zarr_io.py +98 -137
  72. hpcflow/sdk/data/workflow_spec_schema.yaml +2 -0
  73. hpcflow/sdk/demo/cli.py +53 -33
  74. hpcflow/sdk/helper/cli.py +18 -15
  75. hpcflow/sdk/helper/helper.py +75 -63
  76. hpcflow/sdk/helper/watcher.py +61 -28
  77. hpcflow/sdk/log.py +122 -71
  78. hpcflow/sdk/persistence/__init__.py +8 -31
  79. hpcflow/sdk/persistence/base.py +1360 -606
  80. hpcflow/sdk/persistence/defaults.py +6 -0
  81. hpcflow/sdk/persistence/discovery.py +38 -0
  82. hpcflow/sdk/persistence/json.py +568 -188
  83. hpcflow/sdk/persistence/pending.py +382 -179
  84. hpcflow/sdk/persistence/store_resource.py +39 -23
  85. hpcflow/sdk/persistence/types.py +318 -0
  86. hpcflow/sdk/persistence/utils.py +14 -11
  87. hpcflow/sdk/persistence/zarr.py +1337 -433
  88. hpcflow/sdk/runtime.py +44 -41
  89. hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
  90. hpcflow/sdk/submission/jobscript.py +1651 -692
  91. hpcflow/sdk/submission/schedulers/__init__.py +167 -39
  92. hpcflow/sdk/submission/schedulers/direct.py +121 -81
  93. hpcflow/sdk/submission/schedulers/sge.py +170 -129
  94. hpcflow/sdk/submission/schedulers/slurm.py +291 -268
  95. hpcflow/sdk/submission/schedulers/utils.py +12 -2
  96. hpcflow/sdk/submission/shells/__init__.py +14 -15
  97. hpcflow/sdk/submission/shells/base.py +150 -29
  98. hpcflow/sdk/submission/shells/bash.py +283 -173
  99. hpcflow/sdk/submission/shells/os_version.py +31 -30
  100. hpcflow/sdk/submission/shells/powershell.py +228 -170
  101. hpcflow/sdk/submission/submission.py +1014 -335
  102. hpcflow/sdk/submission/types.py +140 -0
  103. hpcflow/sdk/typing.py +182 -12
  104. hpcflow/sdk/utils/arrays.py +71 -0
  105. hpcflow/sdk/utils/deferred_file.py +55 -0
  106. hpcflow/sdk/utils/hashing.py +16 -0
  107. hpcflow/sdk/utils/patches.py +12 -0
  108. hpcflow/sdk/utils/strings.py +33 -0
  109. hpcflow/tests/api/test_api.py +32 -0
  110. hpcflow/tests/conftest.py +27 -6
  111. hpcflow/tests/data/multi_path_sequences.yaml +29 -0
  112. hpcflow/tests/data/workflow_test_run_abort.yaml +34 -35
  113. hpcflow/tests/schedulers/sge/test_sge_submission.py +36 -0
  114. hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
  115. hpcflow/tests/scripts/test_input_file_generators.py +282 -0
  116. hpcflow/tests/scripts/test_main_scripts.py +866 -85
  117. hpcflow/tests/scripts/test_non_snippet_script.py +46 -0
  118. hpcflow/tests/scripts/test_ouput_file_parsers.py +353 -0
  119. hpcflow/tests/shells/wsl/test_wsl_submission.py +12 -4
  120. hpcflow/tests/unit/test_action.py +262 -75
  121. hpcflow/tests/unit/test_action_rule.py +9 -4
  122. hpcflow/tests/unit/test_app.py +33 -6
  123. hpcflow/tests/unit/test_cache.py +46 -0
  124. hpcflow/tests/unit/test_cli.py +134 -1
  125. hpcflow/tests/unit/test_command.py +71 -54
  126. hpcflow/tests/unit/test_config.py +142 -16
  127. hpcflow/tests/unit/test_config_file.py +21 -18
  128. hpcflow/tests/unit/test_element.py +58 -62
  129. hpcflow/tests/unit/test_element_iteration.py +50 -1
  130. hpcflow/tests/unit/test_element_set.py +29 -19
  131. hpcflow/tests/unit/test_group.py +4 -2
  132. hpcflow/tests/unit/test_input_source.py +116 -93
  133. hpcflow/tests/unit/test_input_value.py +29 -24
  134. hpcflow/tests/unit/test_jobscript_unit.py +757 -0
  135. hpcflow/tests/unit/test_json_like.py +44 -35
  136. hpcflow/tests/unit/test_loop.py +1396 -84
  137. hpcflow/tests/unit/test_meta_task.py +325 -0
  138. hpcflow/tests/unit/test_multi_path_sequences.py +229 -0
  139. hpcflow/tests/unit/test_object_list.py +17 -12
  140. hpcflow/tests/unit/test_parameter.py +29 -7
  141. hpcflow/tests/unit/test_persistence.py +237 -42
  142. hpcflow/tests/unit/test_resources.py +20 -18
  143. hpcflow/tests/unit/test_run.py +117 -6
  144. hpcflow/tests/unit/test_run_directories.py +29 -0
  145. hpcflow/tests/unit/test_runtime.py +2 -1
  146. hpcflow/tests/unit/test_schema_input.py +23 -15
  147. hpcflow/tests/unit/test_shell.py +23 -2
  148. hpcflow/tests/unit/test_slurm.py +8 -7
  149. hpcflow/tests/unit/test_submission.py +38 -89
  150. hpcflow/tests/unit/test_task.py +352 -247
  151. hpcflow/tests/unit/test_task_schema.py +33 -20
  152. hpcflow/tests/unit/test_utils.py +9 -11
  153. hpcflow/tests/unit/test_value_sequence.py +15 -12
  154. hpcflow/tests/unit/test_workflow.py +114 -83
  155. hpcflow/tests/unit/test_workflow_template.py +0 -1
  156. hpcflow/tests/unit/utils/test_arrays.py +40 -0
  157. hpcflow/tests/unit/utils/test_deferred_file_writer.py +34 -0
  158. hpcflow/tests/unit/utils/test_hashing.py +65 -0
  159. hpcflow/tests/unit/utils/test_patches.py +5 -0
  160. hpcflow/tests/unit/utils/test_redirect_std.py +50 -0
  161. hpcflow/tests/workflows/__init__.py +0 -0
  162. hpcflow/tests/workflows/test_directory_structure.py +31 -0
  163. hpcflow/tests/workflows/test_jobscript.py +334 -1
  164. hpcflow/tests/workflows/test_run_status.py +198 -0
  165. hpcflow/tests/workflows/test_skip_downstream.py +696 -0
  166. hpcflow/tests/workflows/test_submission.py +140 -0
  167. hpcflow/tests/workflows/test_workflows.py +160 -15
  168. hpcflow/tests/workflows/test_zip.py +18 -0
  169. hpcflow/viz_demo.ipynb +6587 -3
  170. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/METADATA +8 -4
  171. hpcflow_new2-0.2.0a199.dist-info/RECORD +221 -0
  172. hpcflow/sdk/core/parallel.py +0 -21
  173. hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
  174. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/LICENSE +0 -0
  175. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/WHEEL +0 -0
  176. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/entry_points.txt +0 -0
@@ -1,58 +1,92 @@
1
1
  """
2
- Shell models based on the Bourne-Again Shell.
2
+ Shell models based on the GNU Bourne-Again Shell.
3
3
  """
4
4
 
5
+ from __future__ import annotations
5
6
  from pathlib import Path
6
7
  import subprocess
7
8
  from textwrap import dedent, indent
8
- from typing import Dict, List, Optional, Union
9
+ from typing import TYPE_CHECKING
10
+ from typing_extensions import override
11
+ from hpcflow.sdk.typing import hydrate
9
12
  from hpcflow.sdk.core import ABORT_EXIT_CODE
10
- from hpcflow.sdk.submission.shells import Shell
13
+ from hpcflow.sdk.submission.shells.base import Shell
11
14
  from hpcflow.sdk.submission.shells.os_version import (
12
15
  get_OS_info_POSIX,
13
16
  get_OS_info_windows,
14
17
  )
15
18
 
19
+ if TYPE_CHECKING:
20
+ from collections.abc import Mapping
21
+ from typing import Any, ClassVar
22
+ from .base import VersionInfo, JobscriptHeaderArgs
16
23
 
24
+
25
+ @hydrate
17
26
  class Bash(Shell):
18
27
  """
19
28
  Class to represent using bash on a POSIX OS to generate and submit a jobscript.
20
29
  """
21
30
 
22
31
  #: Default for executable name.
23
- DEFAULT_EXE = "/bin/bash"
32
+ DEFAULT_EXE: ClassVar[str] = "/bin/bash"
24
33
 
25
34
  #: File extension for jobscripts.
26
- JS_EXT = ".sh"
35
+ JS_EXT: ClassVar[str] = ".sh"
27
36
  #: Basic indent.
28
- JS_INDENT = " "
37
+ JS_INDENT: ClassVar[str] = " "
29
38
  #: Indent for environment setup.
30
- JS_ENV_SETUP_INDENT = 2 * JS_INDENT
39
+ JS_ENV_SETUP_INDENT: ClassVar[str] = 2 * JS_INDENT
31
40
  #: Template for the jobscript shebang line.
32
- JS_SHEBANG = """#!{shebang_executable} {shebang_args}"""
33
- #: Template for the common part of the jobscript header.
34
- JS_HEADER = dedent(
41
+ JS_SHEBANG: ClassVar[str] = """#!{shebang_executable} {shebang_args}"""
42
+ #: Template for the jobscript functions file.
43
+ JS_FUNCS: ClassVar[str] = dedent(
35
44
  """\
36
45
  {workflow_app_alias} () {{
37
46
  (
38
47
  {env_setup}{app_invoc}\\
39
- --with-config log_file_path "`pwd`/{run_log_file}"\\
48
+ --with-config log_file_path "${app_caps}_LOG_PATH"\\
40
49
  --config-dir "{config_dir}"\\
41
50
  --config-key "{config_invoc_key}"\\
42
51
  "$@"
43
52
  )
44
53
  }}
45
-
54
+ """
55
+ )
56
+ #: Template for the common part of the jobscript header.
57
+ JS_HEADER: ClassVar[str] = dedent(
58
+ """\
46
59
  WK_PATH=`pwd`
47
60
  WK_PATH_ARG="$WK_PATH"
48
61
  SUB_IDX={sub_idx}
49
62
  JS_IDX={js_idx}
50
- EAR_ID_FILE="$WK_PATH/artifacts/submissions/${{SUB_IDX}}/{EAR_file_name}"
51
- ELEM_RUN_DIR_FILE="$WK_PATH/artifacts/submissions/${{SUB_IDX}}/{element_run_dirs_file_path}"
63
+ APP_CAPS={app_caps}
64
+
65
+ SUB_DIR="$WK_PATH/artifacts/submissions/${{SUB_IDX}}"
66
+ JS_FUNCS_PATH="$SUB_DIR/{jobscript_functions_dir}/{jobscript_functions_name}"
67
+ . "$JS_FUNCS_PATH"
68
+
69
+ EAR_ID_FILE="$WK_PATH/artifacts/submissions/${{SUB_IDX}}/{run_IDs_file_dir}/{run_IDs_file_name}"
70
+ SUB_TMP_DIR="$SUB_DIR/{tmp_dir_name}"
71
+ SUB_LOG_DIR="$SUB_DIR/{log_dir_name}"
72
+ SUB_STD_DIR="$SUB_DIR/{app_std_dir_name}"
73
+ SUB_SCRIPTS_DIR="$SUB_DIR/{scripts_dir_name}"
74
+
75
+ export {app_caps}_WK_PATH=$WK_PATH
76
+ export {app_caps}_WK_PATH_ARG=$WK_PATH_ARG
77
+ export {app_caps}_SUB_IDX={sub_idx}
78
+ export {app_caps}_SUB_SCRIPTS_DIR=$SUB_SCRIPTS_DIR
79
+ export {app_caps}_SUB_TMP_DIR=$SUB_TMP_DIR
80
+ export {app_caps}_SUB_LOG_DIR=$SUB_LOG_DIR
81
+ export {app_caps}_SUB_STD_DIR=$SUB_STD_DIR
82
+ export {app_caps}_LOG_PATH="$SUB_LOG_DIR/js_${{JS_IDX}}.log"
83
+ export {app_caps}_JS_FUNCS_PATH=$JS_FUNCS_PATH
84
+ export {app_caps}_JS_IDX={js_idx}
85
+ export {app_caps}_RUN_ID_FILE=$EAR_ID_FILE
52
86
  """
53
87
  )
54
88
  #: Template for the jobscript header when scheduled.
55
- JS_SCHEDULER_HEADER = dedent(
89
+ JS_SCHEDULER_HEADER: ClassVar[str] = dedent(
56
90
  """\
57
91
  {shebang}
58
92
 
@@ -61,96 +95,136 @@ class Bash(Shell):
61
95
  """
62
96
  )
63
97
  #: Template for the jobscript header when directly executed.
64
- JS_DIRECT_HEADER = dedent(
98
+ JS_DIRECT_HEADER: ClassVar[str] = dedent(
65
99
  """\
66
100
  {shebang}
67
-
68
101
  {header}
69
102
  {wait_command}
70
103
  """
71
104
  )
72
- #: Template for the jobscript body.
73
- JS_MAIN = dedent(
105
+ #: Template for enabling writing of the app log.
106
+ JS_RUN_LOG_PATH_ENABLE: ClassVar[str] = '"$SUB_LOG_DIR/{run_log_file_name}"'
107
+ #: Template for disabling writing of the app log.
108
+ JS_RUN_LOG_PATH_DISABLE: ClassVar[str] = '" "'
109
+ #: Template for the run execution command.
110
+ JS_RUN_CMD: ClassVar[str] = (
111
+ '{workflow_app_alias} internal workflow "$WK_PATH_ARG" execute-run '
112
+ "$SUB_IDX $JS_IDX $block_idx $block_act_idx $EAR_ID\n"
113
+ )
114
+ #: Template for the execution command for multiple combined runs.
115
+ JS_RUN_CMD_COMBINED: ClassVar[str] = (
116
+ '{workflow_app_alias} internal workflow "$WK_PATH_ARG" execute-combined-runs '
117
+ "$SUB_IDX $JS_IDX\n"
118
+ )
119
+ #: Template for setting up run environment variables and executing the run.
120
+ JS_RUN: ClassVar[str] = dedent(
74
121
  """\
122
+ EAR_ID="$(cut -d'{EAR_files_delimiter}' -f $(($block_act_idx + 1)) <<< $elem_EAR_IDs)"
123
+ if [ "$EAR_ID" = "-1" ]; then
124
+ continue
125
+ fi
126
+
127
+ export {app_caps}_RUN_ID=$EAR_ID
128
+ export {app_caps}_RUN_LOG_PATH={run_log_enable_disable}
129
+ export {app_caps}_LOG_PATH="${app_caps}_RUN_LOG_PATH"
130
+ export {app_caps}_RUN_STD_PATH="$SUB_STD_DIR/${app_caps}_RUN_ID.txt"
131
+ export {app_caps}_BLOCK_ACT_IDX=$block_act_idx
132
+
133
+ cd "$SUB_TMP_DIR"
134
+
135
+ {run_cmd}
136
+ """
137
+ )
138
+ #: Template for the action-run processing loop in a jobscript.
139
+ JS_ACT_MULTI: ClassVar[str] = dedent(
140
+ """\
141
+ for ((block_act_idx=0;block_act_idx<{num_actions};block_act_idx++))
142
+ do
143
+ {run_block}
144
+ done
145
+ """
146
+ )
147
+ #: Template for the single-action-run execution in a jobscript.
148
+ JS_ACT_SINGLE: ClassVar[str] = dedent(
149
+ """\
150
+ block_act_idx=0
151
+ {run_block}
152
+ """
153
+ )
154
+ #: Template for setting up environment variables and running one or more action-runs.
155
+ JS_MAIN: ClassVar[str] = dedent(
156
+ """\
157
+ block_elem_idx=$(( $JS_elem_idx - {block_start_elem_idx} ))
75
158
  elem_EAR_IDs=`sed "$((${{JS_elem_idx}} + 1))q;d" "$EAR_ID_FILE"`
76
- elem_run_dirs=`sed "$((${{JS_elem_idx}} + 1))q;d" "$ELEM_RUN_DIR_FILE"`
77
-
78
- for ((JS_act_idx=0;JS_act_idx<{num_actions};JS_act_idx++))
79
- do
80
-
81
- EAR_ID="$(cut -d'{EAR_files_delimiter}' -f $(($JS_act_idx + 1)) <<< $elem_EAR_IDs)"
82
- if [ "$EAR_ID" = "-1" ]; then
83
- continue
84
- fi
85
-
86
- run_dir="$(cut -d'{EAR_files_delimiter}' -f $(($JS_act_idx + 1)) <<< $elem_run_dirs)"
87
- cd "$WK_PATH/$run_dir"
88
- app_stream_file="`pwd`/{run_stream_file}"
89
-
90
- skip=`{workflow_app_alias} internal workflow "$WK_PATH_ARG" get-ear-skipped $EAR_ID 2>> "$app_stream_file"`
91
- exc_sk=$?
92
-
93
- if [ $exc_sk -eq 0 ]; then
94
-
95
- if [ "$skip" = "1" ]; then
96
- continue
97
- fi
98
-
99
- {workflow_app_alias} internal workflow "$WK_PATH_ARG" write-commands $SUB_IDX $JS_IDX $JS_act_idx $EAR_ID >> "$app_stream_file" 2>&1
100
- exc_wc=$?
101
-
102
- {workflow_app_alias} internal workflow "$WK_PATH_ARG" set-ear-start $EAR_ID >> "$app_stream_file" 2>&1
103
- exc_se=$?
104
-
105
- if [ $exc_wc -eq 0 ] && [ $exc_se -eq 0 ]; then
106
- . {commands_file_name}
107
- exit_code=$?
108
- else
109
- exit_code=$([ $exc_wc -ne 0 ] && echo "$exc_wc" || echo "$exc_se")
110
- fi
111
-
112
- else
113
- exit_code=$exc_sk
114
- fi
115
-
116
- {workflow_app_alias} internal workflow "$WK_PATH_ARG" set-ear-end $JS_IDX $JS_act_idx $EAR_ID "--" "$exit_code" >> "$app_stream_file" 2>&1
117
-
118
- done
159
+ export {app_caps}_JS_ELEM_IDX=$JS_elem_idx
160
+ export {app_caps}_BLOCK_ELEM_IDX=$block_elem_idx
161
+
162
+ {action}
163
+ """
164
+ )
165
+ #: Template for a jobscript-block header.
166
+ JS_BLOCK_HEADER: ClassVar[str] = dedent( # for single-block jobscripts only
167
+ """\
168
+ block_idx=0
169
+ export {app_caps}_BLOCK_IDX=0
170
+ """
171
+ )
172
+ #: Template for single-element execution.
173
+ JS_ELEMENT_SINGLE: ClassVar[str] = dedent(
174
+ """\
175
+ JS_elem_idx={block_start_elem_idx}
176
+ {main}
119
177
  """
120
178
  )
121
179
  #: Template for the element processing loop in a jobscript.
122
- JS_ELEMENT_LOOP = dedent(
180
+ JS_ELEMENT_MULTI_LOOP: ClassVar[str] = dedent(
123
181
  """\
124
- for ((JS_elem_idx=0;JS_elem_idx<{num_elements};JS_elem_idx++))
182
+ for ((JS_elem_idx={block_start_elem_idx};JS_elem_idx<$(({block_start_elem_idx} + {num_elements}));JS_elem_idx++))
125
183
  do
126
184
  {main}
127
185
  done
128
- cd "$WK_PATH"
129
186
  """
130
187
  )
131
188
  #: Template for the array handling code in a jobscript.
132
- JS_ELEMENT_ARRAY = dedent(
189
+ JS_ELEMENT_MULTI_ARRAY: ClassVar[str] = dedent(
133
190
  """\
134
191
  JS_elem_idx=$(({scheduler_array_item_var} - 1))
135
192
  {main}
136
- cd "$WK_PATH"
137
193
  """
138
194
  )
139
-
140
- def __init__(self, *args, **kwargs):
141
- super().__init__(*args, **kwargs)
195
+ #: Template for the jobscript block loop in a jobscript.
196
+ JS_BLOCK_LOOP: ClassVar[str] = dedent(
197
+ """\
198
+ num_elements={num_elements}
199
+ num_actions={num_actions}
200
+ block_start_elem_idx=0
201
+ for ((block_idx=0;block_idx<{num_blocks};block_idx++))
202
+ do
203
+ export {app_caps}_BLOCK_IDX=$block_idx
204
+ {element_loop}
205
+ block_start_elem_idx=$(($block_start_elem_idx + ${{num_elements[$block_idx]}}))
206
+ done
207
+ """
208
+ )
209
+ #: Template for the jobscript footer.
210
+ JS_FOOTER: ClassVar[str] = dedent(
211
+ """\
212
+ cd $WK_PATH
213
+ """
214
+ )
142
215
 
143
216
  @property
144
- def linux_release_file(self):
217
+ def linux_release_file(self) -> str:
145
218
  """
146
219
  The name of the file describing the Linux version.
147
220
  """
148
221
  return self.os_args["linux_release_file"]
149
222
 
150
- def _get_OS_info_POSIX(self):
223
+ def _get_OS_info_POSIX(self) -> Mapping[str, str]:
151
224
  return get_OS_info_POSIX(linux_release_file=self.linux_release_file)
152
225
 
153
- def get_version_info(self, exclude_os: Optional[bool] = False) -> Dict:
226
+ @override
227
+ def get_version_info(self, exclude_os: bool = False) -> VersionInfo:
154
228
  """Get bash version information.
155
229
 
156
230
  Parameters
@@ -171,37 +245,74 @@ class Bash(Shell):
171
245
  else:
172
246
  raise RuntimeError("Failed to parse bash version information.")
173
247
 
174
- out = {
248
+ return {
175
249
  "shell_name": "bash",
176
250
  "shell_executable": self.executable,
177
251
  "shell_version": bash_version,
252
+ **({} if exclude_os else self._get_OS_info_POSIX()),
178
253
  }
179
254
 
180
- if not exclude_os:
181
- out.update(**self._get_OS_info_POSIX())
182
-
183
- return out
184
-
185
255
  @staticmethod
186
- def process_app_invoc_executable(app_invoc_exe):
256
+ def process_app_invoc_executable(app_invoc_exe: str) -> str:
187
257
  # escape spaces with a back slash:
188
- app_invoc_exe = app_invoc_exe.replace(" ", r"\ ")
189
- return app_invoc_exe
258
+ return app_invoc_exe.replace(" ", r"\ ")
259
+
260
+ @override
261
+ def format_env_var_get(self, var: str) -> str:
262
+ """
263
+ Format retrieval of a shell environment variable.
264
+ """
265
+ return f"${var}"
266
+
267
+ @override
268
+ def format_array(self, lst: list) -> str:
269
+ """
270
+ Format construction of a shell array.
271
+ """
272
+ return "(" + " ".join(str(i) for i in lst) + ")"
190
273
 
191
- def format_stream_assignment(self, shell_var_name, command):
274
+ @override
275
+ def format_array_get_item(self, arr_name: str, index: int | str) -> str:
276
+ """
277
+ Format retrieval of a shell array item at a specified index.
278
+ """
279
+ return f"${{{arr_name}[{index}]}}"
280
+
281
+ @override
282
+ def format_stream_assignment(self, shell_var_name: str, command: str) -> str:
192
283
  """
193
284
  Produce code to assign the output of the command to a shell variable.
194
285
  """
195
286
  return f"{shell_var_name}=`{command}`"
196
287
 
288
+ @override
289
+ def format_source_functions_file(self, app_name: str, commands: str) -> str:
290
+ """
291
+ Format sourcing (i.e. invocation) of the jobscript functions file.
292
+ """
293
+ return dedent(
294
+ """\
295
+ . "${app_caps}_JS_FUNCS_PATH"
296
+
297
+ """
298
+ ).format(app_caps=app_name.upper())
299
+
300
+ @override
301
+ def format_commands_file(self, app_name: str, commands: str) -> str:
302
+ """
303
+ Format the commands file.
304
+ """
305
+ return self.format_source_functions_file(app_name, commands) + commands
306
+
307
+ @override
197
308
  def format_save_parameter(
198
309
  self,
199
310
  workflow_app_alias: str,
200
311
  param_name: str,
201
312
  shell_var_name: str,
202
- EAR_ID: int,
203
313
  cmd_idx: int,
204
314
  stderr: bool,
315
+ app_name: str,
205
316
  ):
206
317
  """
207
318
  Produce code to save a parameter's value into the workflow persistent store.
@@ -209,72 +320,14 @@ class Bash(Shell):
209
320
  # TODO: quote shell_var_name as well? e.g. if it's a white-space delimited list?
210
321
  # and test.
211
322
  stderr_str = " --stderr" if stderr else ""
323
+ app_caps = app_name.upper()
212
324
  return (
213
- f"{workflow_app_alias} "
214
- f'internal workflow "$WK_PATH_ARG" save-parameter '
215
- f"{param_name} ${shell_var_name} {EAR_ID} {cmd_idx}{stderr_str} "
216
- f'>> "$app_stream_file" 2>&1'
325
+ f'{workflow_app_alias} --std-stream "${app_caps}_RUN_STD_PATH" '
326
+ f'internal workflow "${app_caps}_WK_PATH_ARG" save-parameter {stderr_str}'
327
+ f'"--" {param_name} ${shell_var_name} ${app_caps}_RUN_ID {cmd_idx}'
217
328
  f"\n"
218
329
  )
219
330
 
220
- def format_loop_check(self, workflow_app_alias: str, loop_name: str, run_ID: int):
221
- """
222
- Produce code to check the looping status of part of a workflow.
223
- """
224
- return (
225
- f"{workflow_app_alias} "
226
- f'internal workflow "$WK_PATH_ARG" check-loop '
227
- f"{loop_name} {run_ID} "
228
- f'>> "$app_stream_file" 2>&1'
229
- f"\n"
230
- )
231
-
232
- def wrap_in_subshell(self, commands: str, abortable: bool) -> str:
233
- """Format commands to run within a subshell.
234
-
235
- This assumes commands ends in a newline.
236
-
237
- """
238
- commands = indent(commands, self.JS_INDENT)
239
- if abortable:
240
- # run commands in the background, and poll a file to check for abort requests:
241
- return dedent(
242
- """\
243
- (
244
- {commands}) &
245
-
246
- pid=$!
247
- abort_file=$WK_PATH/artifacts/submissions/$SUB_IDX/abort_EARs.txt
248
- while true
249
- do
250
- is_abort=`sed "$(($EAR_ID + 1))q;d" $abort_file`
251
- ps -p $pid > /dev/null
252
- if [ $? == 1 ]; then
253
- wait $pid
254
- exitcode=$?
255
- break
256
- elif [ "$is_abort" = "1" ]; then
257
- echo "Abort instruction received; stopping commands..." >> "$app_stream_file"
258
- kill $pid
259
- wait $pid 2>/dev/null
260
- exitcode={abort_exit_code}
261
- break
262
- else
263
- sleep 1 # TODO: TEMP: increase for production
264
- fi
265
- done
266
- return $exitcode
267
- """
268
- ).format(commands=commands, abort_exit_code=ABORT_EXIT_CODE)
269
- else:
270
- # run commands in "foreground":
271
- return dedent(
272
- """\
273
- (
274
- {commands})
275
- """
276
- ).format(commands=commands)
277
-
278
331
 
279
332
  class WSLBash(Bash):
280
333
  """
@@ -282,38 +335,87 @@ class WSLBash(Bash):
282
335
  """
283
336
 
284
337
  #: Default name of the WSL interface executable.
285
- DEFAULT_WSL_EXE = "wsl"
338
+ DEFAULT_WSL_EXE: ClassVar[str] = "wsl"
286
339
 
340
+ #: Template for the jobscript functions file.
341
+ JS_FUNCS: ClassVar[str] = dedent(
342
+ """\
343
+ {{workflow_app_alias}} () {{{{
344
+ (
345
+ {log_path_block}
346
+ {{env_setup}}{{app_invoc}}\\
347
+ --with-config log_file_path "$LOG_FILE_PATH"\\
348
+ --config-dir "{{config_dir}}"\\
349
+ --config-key "{{config_invoc_key}}"\\
350
+ "$@"
351
+ )
352
+ }}}}
353
+ """
354
+ ).format(
355
+ log_path_block=indent(
356
+ dedent(
357
+ """\
358
+ if [ -z "${app_caps}_LOG_PATH" ] || [ "${app_caps}_LOG_PATH" = " " ]; then
359
+ LOG_FILE_PATH=" "
360
+ else
361
+ LOG_FILE_PATH="$(wslpath -m ${app_caps}_LOG_PATH)"
362
+ fi
363
+ """
364
+ ),
365
+ prefix=Bash.JS_ENV_SETUP_INDENT,
366
+ )
367
+ )
287
368
  #: Template for the common part of the jobscript header.
288
- JS_HEADER = Bash.JS_HEADER.replace(
369
+ JS_HEADER: ClassVar[str] = Bash.JS_HEADER.replace(
289
370
  'WK_PATH_ARG="$WK_PATH"',
290
371
  'WK_PATH_ARG=`wslpath -m "$WK_PATH"`',
291
- ).replace(
292
- '--with-config log_file_path "`pwd`',
293
- '--with-config log_file_path "$(wslpath -m `pwd`)',
372
+ )
373
+ #: Template for the run execution command.
374
+ JS_RUN_CMD: ClassVar[str] = (
375
+ dedent(
376
+ """\
377
+ WSLENV=$WSLENV:${{APP_CAPS}}_WK_PATH
378
+ WSLENV=$WSLENV:${{APP_CAPS}}_WK_PATH_ARG
379
+ WSLENV=$WSLENV:${{APP_CAPS}}_JS_FUNCS_PATH
380
+ WSLENV=$WSLENV:${{APP_CAPS}}_STD_STREAM_FILE
381
+ WSLENV=$WSLENV:${{APP_CAPS}}_SUB_IDX
382
+ WSLENV=$WSLENV:${{APP_CAPS}}_JS_IDX
383
+ WSLENV=$WSLENV:${{APP_CAPS}}_RUN_ID
384
+ WSLENV=$WSLENV:${{APP_CAPS}}_BLOCK_ACT_IDX
385
+ WSLENV=$WSLENV:${{APP_CAPS}}_JS_ELEM_IDX
386
+ WSLENV=$WSLENV:${{APP_CAPS}}_BLOCK_ELEM_IDX
387
+ WSLENV=$WSLENV:${{APP_CAPS}}_BLOCK_IDX
388
+ WSLENV=$WSLENV:${{APP_CAPS}}_LOG_PATH/p
389
+
390
+ """
391
+ )
392
+ + Bash.JS_RUN_CMD
294
393
  )
295
394
 
296
395
  def __init__(
297
396
  self,
298
- WSL_executable: Optional[str] = None,
299
- WSL_distribution: Optional[str] = None,
300
- WSL_user: Optional[str] = None,
397
+ WSL_executable: str | None = None,
398
+ WSL_distribution: str | None = None,
399
+ WSL_user: str | None = None,
301
400
  *args,
302
401
  **kwargs,
303
402
  ):
403
+ #: The WSL executable wrapper.
304
404
  self.WSL_executable = WSL_executable or self.DEFAULT_WSL_EXE
405
+ #: The WSL distribution to use, if any.
305
406
  self.WSL_distribution = WSL_distribution
407
+ #: The WSL user to use, if any.
306
408
  self.WSL_user = WSL_user
307
409
  super().__init__(*args, **kwargs)
308
410
 
309
- def __eq__(self, other) -> bool:
411
+ def __eq__(self, other: Any) -> bool:
310
412
  return super().__eq__(other) and (
311
413
  self.WSL_executable == other.WSL_executable
312
414
  and self.WSL_distribution == other.WSL_distribution
313
415
  and self.WSL_user == other.WSL_user
314
416
  )
315
417
 
316
- def _get_WSL_command(self):
418
+ def _get_WSL_command(self) -> list[str]:
317
419
  out = [self.WSL_executable]
318
420
  if self.WSL_distribution:
319
421
  out += ["--distribution", self.WSL_distribution]
@@ -322,14 +424,14 @@ class WSLBash(Bash):
322
424
  return out
323
425
 
324
426
  @property
325
- def executable(self) -> List[str]:
427
+ def executable(self) -> list[str]:
326
428
  return self._get_WSL_command() + super().executable
327
429
 
328
430
  @property
329
- def shebang_executable(self) -> List[str]:
431
+ def shebang_executable(self) -> list[str]:
330
432
  return super().executable
331
433
 
332
- def _get_OS_info_POSIX(self):
434
+ def _get_OS_info_POSIX(self) -> Mapping[str, str]:
333
435
  return get_OS_info_POSIX(
334
436
  WSL_executable=self._get_WSL_command(),
335
437
  use_py=False,
@@ -337,27 +439,29 @@ class WSLBash(Bash):
337
439
  )
338
440
 
339
441
  @staticmethod
340
- def _convert_to_wsl_path(win_path: Union[str, Path]) -> str:
442
+ def _convert_to_wsl_path(win_path: str | Path) -> str:
341
443
  win_path = Path(win_path)
342
444
  parts = list(win_path.parts)
343
445
  parts[0] = f"/mnt/{win_path.drive.lower().rstrip(':')}"
344
- wsl_path = "/".join(parts)
345
- return wsl_path
446
+ return "/".join(parts)
346
447
 
347
- def process_JS_header_args(self, header_args):
448
+ def process_JS_header_args(
449
+ self, header_args: JobscriptHeaderArgs
450
+ ) -> JobscriptHeaderArgs:
348
451
  # convert executable windows paths to posix style as expected by WSL:
349
- header_args["app_invoc"][0] = self._convert_to_wsl_path(
350
- header_args["app_invoc"][0]
351
- )
452
+ ai = header_args["app_invoc"]
453
+ if isinstance(ai, list):
454
+ ai[0] = self._convert_to_wsl_path(ai[0])
352
455
  return super().process_JS_header_args(header_args)
353
456
 
354
457
  def prepare_JS_path(self, js_path: Path) -> str:
355
458
  return self._convert_to_wsl_path(js_path)
356
459
 
357
- def prepare_element_run_dirs(self, run_dirs: List[List[Path]]) -> List[List[str]]:
358
- return [["/".join(str(j).split("\\")) for j in i] for i in run_dirs]
460
+ def prepare_element_run_dirs(self, run_dirs: list[list[Path]]) -> list[list[str]]:
461
+ return [[str(path).replace("\\", "/") for path in i] for i in run_dirs]
359
462
 
360
- def get_version_info(self, exclude_os: Optional[bool] = False) -> Dict:
463
+ @override
464
+ def get_version_info(self, exclude_os: bool = False) -> VersionInfo:
361
465
  """Get WSL and bash version information.
362
466
 
363
467
  Parameters
@@ -368,12 +472,14 @@ class WSLBash(Bash):
368
472
  """
369
473
  vers_info = super().get_version_info(exclude_os=exclude_os)
370
474
 
371
- vers_info["shell_name"] = ("wsl+" + vers_info["shell_name"]).lower()
475
+ vers_info["shell_name"] = f"wsl+{vers_info['shell_name']}".lower()
372
476
  vers_info["WSL_executable"] = self.WSL_executable
373
- vers_info["WSL_distribution"] = self.WSL_distribution
374
- vers_info["WSL_user"] = self.WSL_user
477
+ if self.WSL_distribution:
478
+ vers_info["WSL_distribution"] = self.WSL_distribution
479
+ if self.WSL_user:
480
+ vers_info["WSL_user"] = self.WSL_user
375
481
 
376
- for key in list(vers_info.keys()):
482
+ for key in tuple(vers_info):
377
483
  if key.startswith("OS_"):
378
484
  vers_info[f"WSL_{key}"] = vers_info.pop(key)
379
485
 
@@ -381,3 +487,7 @@ class WSLBash(Bash):
381
487
  vers_info.update(**get_OS_info_windows())
382
488
 
383
489
  return vers_info
490
+
491
+ def get_command_file_launch_command(self, cmd_file_path: str) -> list[str]:
492
+ """Get the command for launching the commands file for a given run."""
493
+ return self.executable + [self._convert_to_wsl_path(cmd_file_path)]