hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. hpcflow/__pyinstaller/hook-hpcflow.py +9 -6
  2. hpcflow/_version.py +1 -1
  3. hpcflow/app.py +1 -0
  4. hpcflow/data/scripts/bad_script.py +2 -0
  5. hpcflow/data/scripts/do_nothing.py +2 -0
  6. hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py +4 -0
  7. hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py +8 -0
  8. hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py +4 -0
  9. hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py +4 -0
  10. hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py +7 -0
  11. hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py +4 -0
  12. hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py +7 -0
  13. hpcflow/data/scripts/input_file_generator_basic.py +3 -0
  14. hpcflow/data/scripts/input_file_generator_basic_FAIL.py +3 -0
  15. hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py +8 -0
  16. hpcflow/data/scripts/main_script_test_direct_in.py +3 -0
  17. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py +6 -0
  18. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py +6 -0
  19. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py +7 -0
  20. hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py +6 -0
  21. hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py +6 -0
  22. hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py +6 -0
  23. hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
  24. hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py +12 -0
  25. hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
  26. hpcflow/data/scripts/main_script_test_json_out_FAIL.py +3 -0
  27. hpcflow/data/scripts/main_script_test_shell_env_vars.py +12 -0
  28. hpcflow/data/scripts/main_script_test_std_out_std_err.py +6 -0
  29. hpcflow/data/scripts/output_file_parser_basic.py +3 -0
  30. hpcflow/data/scripts/output_file_parser_basic_FAIL.py +7 -0
  31. hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py +8 -0
  32. hpcflow/data/scripts/script_exit_test.py +5 -0
  33. hpcflow/data/template_components/environments.yaml +1 -1
  34. hpcflow/sdk/__init__.py +26 -15
  35. hpcflow/sdk/app.py +2192 -768
  36. hpcflow/sdk/cli.py +506 -296
  37. hpcflow/sdk/cli_common.py +105 -7
  38. hpcflow/sdk/config/__init__.py +1 -1
  39. hpcflow/sdk/config/callbacks.py +115 -43
  40. hpcflow/sdk/config/cli.py +126 -103
  41. hpcflow/sdk/config/config.py +674 -318
  42. hpcflow/sdk/config/config_file.py +131 -95
  43. hpcflow/sdk/config/errors.py +125 -84
  44. hpcflow/sdk/config/types.py +148 -0
  45. hpcflow/sdk/core/__init__.py +25 -1
  46. hpcflow/sdk/core/actions.py +1771 -1059
  47. hpcflow/sdk/core/app_aware.py +24 -0
  48. hpcflow/sdk/core/cache.py +139 -79
  49. hpcflow/sdk/core/command_files.py +263 -287
  50. hpcflow/sdk/core/commands.py +145 -112
  51. hpcflow/sdk/core/element.py +828 -535
  52. hpcflow/sdk/core/enums.py +192 -0
  53. hpcflow/sdk/core/environment.py +74 -93
  54. hpcflow/sdk/core/errors.py +455 -52
  55. hpcflow/sdk/core/execute.py +207 -0
  56. hpcflow/sdk/core/json_like.py +540 -272
  57. hpcflow/sdk/core/loop.py +751 -347
  58. hpcflow/sdk/core/loop_cache.py +164 -47
  59. hpcflow/sdk/core/object_list.py +370 -207
  60. hpcflow/sdk/core/parameters.py +1100 -627
  61. hpcflow/sdk/core/rule.py +59 -41
  62. hpcflow/sdk/core/run_dir_files.py +21 -37
  63. hpcflow/sdk/core/skip_reason.py +7 -0
  64. hpcflow/sdk/core/task.py +1649 -1339
  65. hpcflow/sdk/core/task_schema.py +308 -196
  66. hpcflow/sdk/core/test_utils.py +191 -114
  67. hpcflow/sdk/core/types.py +440 -0
  68. hpcflow/sdk/core/utils.py +485 -309
  69. hpcflow/sdk/core/validation.py +82 -9
  70. hpcflow/sdk/core/workflow.py +2544 -1178
  71. hpcflow/sdk/core/zarr_io.py +98 -137
  72. hpcflow/sdk/data/workflow_spec_schema.yaml +2 -0
  73. hpcflow/sdk/demo/cli.py +53 -33
  74. hpcflow/sdk/helper/cli.py +18 -15
  75. hpcflow/sdk/helper/helper.py +75 -63
  76. hpcflow/sdk/helper/watcher.py +61 -28
  77. hpcflow/sdk/log.py +122 -71
  78. hpcflow/sdk/persistence/__init__.py +8 -31
  79. hpcflow/sdk/persistence/base.py +1360 -606
  80. hpcflow/sdk/persistence/defaults.py +6 -0
  81. hpcflow/sdk/persistence/discovery.py +38 -0
  82. hpcflow/sdk/persistence/json.py +568 -188
  83. hpcflow/sdk/persistence/pending.py +382 -179
  84. hpcflow/sdk/persistence/store_resource.py +39 -23
  85. hpcflow/sdk/persistence/types.py +318 -0
  86. hpcflow/sdk/persistence/utils.py +14 -11
  87. hpcflow/sdk/persistence/zarr.py +1337 -433
  88. hpcflow/sdk/runtime.py +44 -41
  89. hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
  90. hpcflow/sdk/submission/jobscript.py +1651 -692
  91. hpcflow/sdk/submission/schedulers/__init__.py +167 -39
  92. hpcflow/sdk/submission/schedulers/direct.py +121 -81
  93. hpcflow/sdk/submission/schedulers/sge.py +170 -129
  94. hpcflow/sdk/submission/schedulers/slurm.py +291 -268
  95. hpcflow/sdk/submission/schedulers/utils.py +12 -2
  96. hpcflow/sdk/submission/shells/__init__.py +14 -15
  97. hpcflow/sdk/submission/shells/base.py +150 -29
  98. hpcflow/sdk/submission/shells/bash.py +283 -173
  99. hpcflow/sdk/submission/shells/os_version.py +31 -30
  100. hpcflow/sdk/submission/shells/powershell.py +228 -170
  101. hpcflow/sdk/submission/submission.py +1014 -335
  102. hpcflow/sdk/submission/types.py +140 -0
  103. hpcflow/sdk/typing.py +182 -12
  104. hpcflow/sdk/utils/arrays.py +71 -0
  105. hpcflow/sdk/utils/deferred_file.py +55 -0
  106. hpcflow/sdk/utils/hashing.py +16 -0
  107. hpcflow/sdk/utils/patches.py +12 -0
  108. hpcflow/sdk/utils/strings.py +33 -0
  109. hpcflow/tests/api/test_api.py +32 -0
  110. hpcflow/tests/conftest.py +27 -6
  111. hpcflow/tests/data/multi_path_sequences.yaml +29 -0
  112. hpcflow/tests/data/workflow_test_run_abort.yaml +34 -35
  113. hpcflow/tests/schedulers/sge/test_sge_submission.py +36 -0
  114. hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
  115. hpcflow/tests/scripts/test_input_file_generators.py +282 -0
  116. hpcflow/tests/scripts/test_main_scripts.py +866 -85
  117. hpcflow/tests/scripts/test_non_snippet_script.py +46 -0
  118. hpcflow/tests/scripts/test_ouput_file_parsers.py +353 -0
  119. hpcflow/tests/shells/wsl/test_wsl_submission.py +12 -4
  120. hpcflow/tests/unit/test_action.py +262 -75
  121. hpcflow/tests/unit/test_action_rule.py +9 -4
  122. hpcflow/tests/unit/test_app.py +33 -6
  123. hpcflow/tests/unit/test_cache.py +46 -0
  124. hpcflow/tests/unit/test_cli.py +134 -1
  125. hpcflow/tests/unit/test_command.py +71 -54
  126. hpcflow/tests/unit/test_config.py +142 -16
  127. hpcflow/tests/unit/test_config_file.py +21 -18
  128. hpcflow/tests/unit/test_element.py +58 -62
  129. hpcflow/tests/unit/test_element_iteration.py +50 -1
  130. hpcflow/tests/unit/test_element_set.py +29 -19
  131. hpcflow/tests/unit/test_group.py +4 -2
  132. hpcflow/tests/unit/test_input_source.py +116 -93
  133. hpcflow/tests/unit/test_input_value.py +29 -24
  134. hpcflow/tests/unit/test_jobscript_unit.py +757 -0
  135. hpcflow/tests/unit/test_json_like.py +44 -35
  136. hpcflow/tests/unit/test_loop.py +1396 -84
  137. hpcflow/tests/unit/test_meta_task.py +325 -0
  138. hpcflow/tests/unit/test_multi_path_sequences.py +229 -0
  139. hpcflow/tests/unit/test_object_list.py +17 -12
  140. hpcflow/tests/unit/test_parameter.py +29 -7
  141. hpcflow/tests/unit/test_persistence.py +237 -42
  142. hpcflow/tests/unit/test_resources.py +20 -18
  143. hpcflow/tests/unit/test_run.py +117 -6
  144. hpcflow/tests/unit/test_run_directories.py +29 -0
  145. hpcflow/tests/unit/test_runtime.py +2 -1
  146. hpcflow/tests/unit/test_schema_input.py +23 -15
  147. hpcflow/tests/unit/test_shell.py +23 -2
  148. hpcflow/tests/unit/test_slurm.py +8 -7
  149. hpcflow/tests/unit/test_submission.py +38 -89
  150. hpcflow/tests/unit/test_task.py +352 -247
  151. hpcflow/tests/unit/test_task_schema.py +33 -20
  152. hpcflow/tests/unit/test_utils.py +9 -11
  153. hpcflow/tests/unit/test_value_sequence.py +15 -12
  154. hpcflow/tests/unit/test_workflow.py +114 -83
  155. hpcflow/tests/unit/test_workflow_template.py +0 -1
  156. hpcflow/tests/unit/utils/test_arrays.py +40 -0
  157. hpcflow/tests/unit/utils/test_deferred_file_writer.py +34 -0
  158. hpcflow/tests/unit/utils/test_hashing.py +65 -0
  159. hpcflow/tests/unit/utils/test_patches.py +5 -0
  160. hpcflow/tests/unit/utils/test_redirect_std.py +50 -0
  161. hpcflow/tests/workflows/__init__.py +0 -0
  162. hpcflow/tests/workflows/test_directory_structure.py +31 -0
  163. hpcflow/tests/workflows/test_jobscript.py +334 -1
  164. hpcflow/tests/workflows/test_run_status.py +198 -0
  165. hpcflow/tests/workflows/test_skip_downstream.py +696 -0
  166. hpcflow/tests/workflows/test_submission.py +140 -0
  167. hpcflow/tests/workflows/test_workflows.py +160 -15
  168. hpcflow/tests/workflows/test_zip.py +18 -0
  169. hpcflow/viz_demo.ipynb +6587 -3
  170. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/METADATA +8 -4
  171. hpcflow_new2-0.2.0a199.dist-info/RECORD +221 -0
  172. hpcflow/sdk/core/parallel.py +0 -21
  173. hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
  174. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/LICENSE +0 -0
  175. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/WHEEL +0 -0
  176. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/entry_points.txt +0 -0
@@ -2,8 +2,19 @@
2
2
  Errors from the workflow system.
3
3
  """
4
4
 
5
+ from __future__ import annotations
5
6
  import os
6
- from typing import Iterable, List
7
+ from collections.abc import Iterable, Mapping, Sequence
8
+ from textwrap import indent
9
+ from typing import Any, TYPE_CHECKING
10
+
11
+ if TYPE_CHECKING:
12
+ from logging import Logger
13
+ from .enums import ParallelMode
14
+ from .object_list import WorkflowLoopList
15
+ from .parameters import InputSource, ValueSequence, SchemaInput
16
+ from .types import ScriptData
17
+ from .task import WorkflowTask
7
18
 
8
19
 
9
20
  class InputValueDuplicateSequenceAddress(ValueError):
@@ -11,36 +22,65 @@ class InputValueDuplicateSequenceAddress(ValueError):
11
22
  An InputValue has the same sequence address twice.
12
23
  """
13
24
 
25
+ # FIXME: never used
26
+
14
27
 
15
28
  class TaskTemplateMultipleSchemaObjectives(ValueError):
16
29
  """
17
30
  A TaskTemplate has multiple objectives.
18
31
  """
19
32
 
33
+ def __init__(self, names: set[str]) -> None:
34
+ super().__init__(
35
+ f"All task schemas used within a task must have the same "
36
+ f"objective, but found multiple objectives: {sorted(names)!r}"
37
+ )
38
+
20
39
 
21
40
  class TaskTemplateUnexpectedInput(ValueError):
22
41
  """
23
42
  A TaskTemplate was given unexpected input.
24
43
  """
25
44
 
45
+ def __init__(self, unexpected_types: set[str]) -> None:
46
+ super().__init__(
47
+ f"The following input parameters are unexpected: {sorted(unexpected_types)!r}"
48
+ )
49
+
26
50
 
27
51
  class TaskTemplateUnexpectedSequenceInput(ValueError):
28
52
  """
29
53
  A TaskTemplate was given an unexpected sequence.
30
54
  """
31
55
 
56
+ def __init__(
57
+ self, inp_type: str, expected_types: set[str], seq: ValueSequence
58
+ ) -> None:
59
+ allowed_str = ", ".join(f'"{in_typ}"' for in_typ in expected_types)
60
+ super().__init__(
61
+ f"The input type {inp_type!r} specified in the following sequence"
62
+ f" path is unexpected: {seq.path!r}. Available input types are: "
63
+ f"{allowed_str}."
64
+ )
65
+
32
66
 
33
67
  class TaskTemplateMultipleInputValues(ValueError):
34
68
  """
35
69
  A TaskTemplate had multiple input values bound over each other.
36
70
  """
37
71
 
72
+ def __init__(self, msg: str) -> None:
73
+ super().__init__(msg)
74
+
38
75
 
39
76
  class InvalidIdentifier(ValueError):
40
77
  """
41
78
  A bad identifier name was given.
42
79
  """
43
80
 
81
+ def __init__(self, name: str) -> None:
82
+ super().__init__(f"Invalid string for identifier: {name!r}")
83
+
44
84
 
45
85
  class MissingInputs(Exception):
46
86
  """
@@ -48,17 +88,16 @@ class MissingInputs(Exception):
48
88
 
49
89
  Parameters
50
90
  ----------
51
- message: str
52
- The message of the exception.
53
- missing_inputs: list[str]
91
+ missing_inputs:
54
92
  The missing inputs.
55
93
  """
56
94
 
57
95
  # TODO: add links to doc pages for common user-exceptions?
58
96
 
59
- def __init__(self, message, missing_inputs) -> None:
60
- self.missing_inputs = missing_inputs
61
- super().__init__(message)
97
+ def __init__(self, missing_inputs: Iterable[str]) -> None:
98
+ self.missing_inputs = tuple(missing_inputs)
99
+ missing_str = ", ".join(map(repr, missing_inputs))
100
+ super().__init__(f"The following inputs have no sources: {missing_str}.")
62
101
 
63
102
 
64
103
  class UnrequiredInputSources(ValueError):
@@ -67,23 +106,23 @@ class UnrequiredInputSources(ValueError):
67
106
 
68
107
  Parameters
69
108
  ----------
70
- message: str
71
- The message of the exception.
72
- unrequired_sources: list
109
+ unrequired_sources:
73
110
  The input sources that were not required.
74
111
  """
75
112
 
76
- def __init__(self, message, unrequired_sources) -> None:
77
- self.unrequired_sources = unrequired_sources
78
- for src in unrequired_sources:
79
- if src.startswith("inputs."):
80
- # reminder about how to specify input sources:
81
- message += (
82
- f" Note that input source keys should not be specified with the "
83
- f"'inputs.' prefix. Did you mean to specify {src[len('inputs.'):]!r} "
84
- f"instead of {src!r}?"
85
- )
86
- break
113
+ def __init__(self, unrequired_sources: Iterable[str]) -> None:
114
+ self.unrequired_sources = frozenset(unrequired_sources)
115
+ message = (
116
+ f"The following input sources are not required but have been specified: "
117
+ f'{", ".join(map(repr, sorted(self.unrequired_sources)))}.'
118
+ )
119
+ if any((bad := src).startswith("inputs.") for src in self.unrequired_sources):
120
+ # reminder about how to specify input sources:
121
+ message += (
122
+ f" Note that input source keys should not be specified with the "
123
+ f"'inputs.' prefix. Did you mean to specify "
124
+ f"{bad.removeprefix('inputs.')!r} instead of {bad!r}?"
125
+ )
87
126
  super().__init__(message)
88
127
 
89
128
 
@@ -93,15 +132,16 @@ class ExtraInputs(Exception):
93
132
 
94
133
  Parameters
95
134
  ----------
96
- message: str
97
- The message of the exception.
98
- extra_inputs: list
135
+ extra_inputs:
99
136
  The extra inputs.
100
137
  """
101
138
 
102
- def __init__(self, message, extra_inputs) -> None:
103
- self.extra_inputs = extra_inputs
104
- super().__init__(message)
139
+ def __init__(self, extra_inputs: set[str]) -> None:
140
+ self.extra_inputs = frozenset(extra_inputs)
141
+ super().__init__(
142
+ f"The following inputs are not required, but have been passed: "
143
+ f'{", ".join(f"{typ!r}" for typ in extra_inputs)}.'
144
+ )
105
145
 
106
146
 
107
147
  class UnavailableInputSource(ValueError):
@@ -109,184 +149,361 @@ class UnavailableInputSource(ValueError):
109
149
  An input source was not available.
110
150
  """
111
151
 
152
+ def __init__(
153
+ self, source: InputSource, path: str, avail: Sequence[InputSource]
154
+ ) -> None:
155
+ super().__init__(
156
+ f"The input source {source.to_string()!r} is not "
157
+ f"available for input path {path!r}. Available "
158
+ f"input sources are: {[src.to_string() for src in avail]}."
159
+ )
160
+
112
161
 
113
162
  class InapplicableInputSourceElementIters(ValueError):
114
163
  """
115
164
  An input source element iteration was inapplicable."""
116
165
 
166
+ def __init__(self, source: InputSource, elem_iters_IDs: Sequence[int] | None) -> None:
167
+ super().__init__(
168
+ f"The specified `element_iters` for input source "
169
+ f"{source.to_string()!r} are not all applicable. "
170
+ f"Applicable element iteration IDs for this input source "
171
+ f"are: {elem_iters_IDs!r}."
172
+ )
173
+
117
174
 
118
175
  class NoCoincidentInputSources(ValueError):
119
176
  """
120
177
  Could not line up input sources to make an actual valid execution.
121
178
  """
122
179
 
180
+ def __init__(self, name: str, task_ref: int) -> None:
181
+ super().__init__(
182
+ f"Task {name!r}: input sources from task {task_ref!r} have "
183
+ f"no coincident applicable element iterations. Consider setting "
184
+ f"the element set (or task) argument "
185
+ f"`allow_non_coincident_task_sources` to `True`, which will "
186
+ f"allow for input sources from the same task to use different "
187
+ f"(non-coinciding) subsets of element iterations from the "
188
+ f"source task."
189
+ )
190
+
123
191
 
124
192
  class TaskTemplateInvalidNesting(ValueError):
125
193
  """
126
194
  Invalid nesting in a task template.
127
195
  """
128
196
 
197
+ def __init__(self, key: str, value: float) -> None:
198
+ super().__init__(
199
+ f"`nesting_order` must be >=0 for all keys, but for key {key!r}, value "
200
+ f"of {value!r} was specified."
201
+ )
202
+
129
203
 
130
204
  class TaskSchemaSpecValidationError(Exception):
131
205
  """
132
206
  A task schema failed to validate.
133
207
  """
134
208
 
209
+ # FIXME: never used
210
+
135
211
 
136
212
  class WorkflowSpecValidationError(Exception):
137
213
  """
138
214
  A workflow failed to validate.
139
215
  """
140
216
 
217
+ # FIXME: never used
218
+
141
219
 
142
220
  class InputSourceValidationError(Exception):
143
221
  """
144
222
  An input source failed to validate.
145
223
  """
146
224
 
225
+ # FIXME: never used
226
+
147
227
 
148
228
  class EnvironmentSpecValidationError(Exception):
149
229
  """
150
230
  An environment specification failed to validate.
151
231
  """
152
232
 
233
+ # FIXME: never used
234
+
153
235
 
154
236
  class ParameterSpecValidationError(Exception):
155
237
  """
156
238
  A parameter specification failed to validate.
157
239
  """
158
240
 
241
+ # FIXME: never used
242
+
159
243
 
160
244
  class FileSpecValidationError(Exception):
161
245
  """
162
246
  A file specification failed to validate.
163
247
  """
164
248
 
249
+ # FIXME: never used
250
+
165
251
 
166
252
  class DuplicateExecutableError(ValueError):
167
253
  """
168
254
  The same executable was present twice in an executable environment.
169
255
  """
170
256
 
257
+ def __init__(self, duplicate_labels: list) -> None:
258
+ super().__init__(
259
+ f"Executables must have unique `label`s within each environment, but "
260
+ f"found label(s) multiple times: {duplicate_labels!r}"
261
+ )
262
+
171
263
 
172
264
  class MissingCompatibleActionEnvironment(Exception):
173
265
  """
174
266
  Could not find a compatible action environment.
175
267
  """
176
268
 
269
+ def __init__(self, msg: str) -> None:
270
+ super().__init__(f"No compatible environment is specified for the {msg}.")
271
+
177
272
 
178
273
  class MissingActionEnvironment(Exception):
179
274
  """
180
275
  Could not find an action environment.
181
276
  """
182
277
 
278
+ # FIXME: never used
279
+
183
280
 
184
281
  class ActionEnvironmentMissingNameError(Exception):
185
282
  """
186
283
  An action environment was missing its name.
187
284
  """
188
285
 
286
+ def __init__(self, environment: Mapping[str, Any]) -> None:
287
+ super().__init__(
288
+ "The action-environment environment specification must include a string "
289
+ "`name` key, or be specified as string that is that name. Provided "
290
+ f"environment key was {environment!r}."
291
+ )
292
+
189
293
 
190
294
  class FromSpecMissingObjectError(Exception):
191
295
  """
192
296
  Missing object when deserialising from specification.
193
297
  """
194
298
 
299
+ # FIXME: never used
300
+
195
301
 
196
302
  class TaskSchemaMissingParameterError(Exception):
197
303
  """
198
304
  Parameter was missing from task schema.
199
305
  """
200
306
 
307
+ # FIXME: never used
308
+
201
309
 
202
310
  class ToJSONLikeChildReferenceError(Exception):
203
311
  """
204
312
  Failed to generate or reference a child object when converting to JSON.
205
313
  """
206
314
 
315
+ # FIXME: never thrown
316
+
207
317
 
208
318
  class InvalidInputSourceTaskReference(Exception):
209
319
  """
210
320
  Invalid input source in task reference.
211
321
  """
212
322
 
323
+ def __init__(self, input_source: InputSource, task_ref: int | None = None) -> None:
324
+ super().__init__(
325
+ f"Input source {input_source.to_string()!r} cannot refer to the "
326
+ f"outputs of its own task!"
327
+ if task_ref is None
328
+ else f"Input source {input_source.to_string()!r} refers to a missing "
329
+ f"or inaccessible task: {task_ref!r}."
330
+ )
331
+
213
332
 
214
333
  class WorkflowNotFoundError(Exception):
215
334
  """
216
335
  Could not find the workflow.
217
336
  """
218
337
 
338
+ def __init__(self, path, fs) -> None:
339
+ super().__init__(
340
+ f"Cannot infer a store format at path {path!r} with file system {fs!r}."
341
+ )
342
+
219
343
 
220
344
  class MalformedWorkflowError(Exception):
221
345
  """
222
346
  Workflow was a malformed document.
223
347
  """
224
348
 
349
+ # TODO: use this class
350
+
225
351
 
226
352
  class ValuesAlreadyPersistentError(Exception):
227
353
  """
228
354
  Trying to make a value persistent that already is so.
229
355
  """
230
356
 
357
+ # FIXME: never used
358
+
231
359
 
232
360
  class MalformedParameterPathError(ValueError):
233
361
  """
234
362
  The path to a parameter was ill-formed.
235
363
  """
236
364
 
365
+ def __init__(self, msg: str) -> None:
366
+ super().__init__(msg)
367
+
237
368
 
238
369
  class MalformedNestingOrderPath(ValueError):
239
370
  """
240
371
  A nesting order path was ill-formed.
241
372
  """
242
373
 
374
+ def __init__(self, k: str, allowed_nesting_paths: Iterable[str]) -> None:
375
+ super().__init__(
376
+ f"Element set: nesting order path {k!r} not understood. Each key in "
377
+ f"`nesting_order` must be start with one of "
378
+ f"{sorted(allowed_nesting_paths)!r}."
379
+ )
380
+
243
381
 
244
382
  class UnknownResourceSpecItemError(ValueError):
245
383
  """
246
384
  A resource specification item was not found.
247
385
  """
248
386
 
387
+ def __init__(self, msg: str) -> None:
388
+ super().__init__(msg)
389
+
249
390
 
250
391
  class WorkflowParameterMissingError(AttributeError):
251
392
  """
252
393
  A parameter to a workflow was missing.
253
394
  """
254
395
 
396
+ # FIXME: never thrown
397
+
255
398
 
256
399
  class WorkflowBatchUpdateFailedError(Exception):
257
400
  """
258
401
  An update to a workflow failed.
259
402
  """
260
403
 
404
+ # FIXME: only throw is commented out?
405
+
261
406
 
262
407
  class WorkflowLimitsError(ValueError):
263
408
  """
264
409
  Workflow hit limits.
265
410
  """
266
411
 
412
+ # FIXME: never used
267
413
 
268
- class UnsetParameterDataError(Exception):
414
+
415
+ class UnsetParameterDataErrorBase(Exception):
416
+ """
417
+ Exceptions related to attempts to retrieve unset parameters.
418
+ """
419
+
420
+
421
+ class UnsetParameterDataError(UnsetParameterDataErrorBase):
269
422
  """
270
423
  Tried to read from an unset parameter.
271
424
  """
272
425
 
426
+ def __init__(self, path: str | None, path_i: str) -> None:
427
+ super().__init__(
428
+ f"Element data path {path!r} resolves to unset data for "
429
+ f"(at least) data-index path: {path_i!r}."
430
+ )
431
+
432
+
433
+ class UnsetParameterFractionLimitExceededError(UnsetParameterDataErrorBase):
434
+ """
435
+ Given the specified `allow_failed_dependencies`, the fraction of failed dependencies
436
+ (unset parameter data) is too high."""
437
+
438
+ def __init__(
439
+ self,
440
+ schema_inp: SchemaInput,
441
+ task: WorkflowTask,
442
+ unset_fraction: float,
443
+ log: Logger | None = None,
444
+ ):
445
+ msg = (
446
+ f"Input {schema_inp.parameter.typ!r} of task {task.name!r}: higher "
447
+ f"proportion of dependencies failed ({unset_fraction!r}) than allowed "
448
+ f"({schema_inp.allow_failed_dependencies!r})."
449
+ )
450
+ if log:
451
+ log.info(msg)
452
+ super().__init__(msg)
453
+
454
+
455
+ class UnsetParameterNumberLimitExceededError(UnsetParameterDataErrorBase):
456
+ """
457
+ Given the specified `allow_failed_dependencies`, the number of failed dependencies
458
+ (unset parameter data) is too high."""
459
+
460
+ def __init__(
461
+ self,
462
+ schema_inp: SchemaInput,
463
+ task: WorkflowTask,
464
+ unset_num: int,
465
+ log: Logger | None = None,
466
+ ):
467
+ msg = (
468
+ f"Input {schema_inp.parameter.typ!r} of task {task.name!r}: higher number of "
469
+ f"dependencies failed ({unset_num!r}) than allowed "
470
+ f"({schema_inp.allow_failed_dependencies!r})."
471
+ )
472
+ if log:
473
+ log.info(msg)
474
+ super().__init__(msg)
475
+
273
476
 
274
477
  class LoopAlreadyExistsError(Exception):
275
478
  """
276
479
  A particular loop (or its name) already exists.
277
480
  """
278
481
 
482
+ def __init__(self, loop_name: str, loops: WorkflowLoopList) -> None:
483
+ super().__init__(
484
+ f"A loop with the name {loop_name!r} already exists in the workflow: "
485
+ f"{getattr(loops, loop_name)!r}."
486
+ )
487
+
279
488
 
280
489
  class LoopTaskSubsetError(ValueError):
281
490
  """
282
491
  Problem constructing a subset of a task for a loop.
283
492
  """
284
493
 
494
+ def __init__(self, loop_name: str, task_indices: Sequence[int]) -> None:
495
+ super().__init__(
496
+ f"Loop {loop_name!r}: task subset must be an ascending contiguous range, "
497
+ f"but specified task indices were: {task_indices!r}."
498
+ )
499
+
285
500
 
286
501
  class SchedulerVersionsFailure(RuntimeError):
287
502
  """We couldn't get the scheduler and or shell versions."""
288
503
 
289
- def __init__(self, message):
504
+ # FIXME: unused
505
+
506
+ def __init__(self, message: str) -> None:
290
507
  self.message = message
291
508
  super().__init__(message)
292
509
 
@@ -298,15 +515,16 @@ class JobscriptSubmissionFailure(RuntimeError):
298
515
 
299
516
  def __init__(
300
517
  self,
301
- message,
302
- submit_cmd,
303
- js_idx,
304
- js_path,
305
- stdout,
306
- stderr,
307
- subprocess_exc,
308
- job_ID_parse_exc,
309
- ) -> None:
518
+ message: str,
519
+ *,
520
+ submit_cmd: list[str],
521
+ js_idx: int,
522
+ js_path: str,
523
+ stdout: str | None = None,
524
+ stderr: str | None = None,
525
+ subprocess_exc: Exception | None = None,
526
+ job_ID_parse_exc: Exception | None = None,
527
+ ):
310
528
  self.message = message
311
529
  self.submit_cmd = submit_cmd
312
530
  self.js_idx = js_idx
@@ -323,9 +541,37 @@ class SubmissionFailure(RuntimeError):
323
541
  A job submission failed.
324
542
  """
325
543
 
326
- def __init__(self, message) -> None:
327
- self.message = message
328
- super().__init__(message)
544
+ def __init__(
545
+ self,
546
+ sub_idx: int,
547
+ submitted_js_idx: Sequence[int],
548
+ exceptions: Iterable[JobscriptSubmissionFailure],
549
+ ) -> None:
550
+ msg = f"Some jobscripts in submission index {sub_idx} could not be submitted"
551
+ if submitted_js_idx:
552
+ msg += f" (but jobscripts {submitted_js_idx} were submitted successfully):"
553
+ else:
554
+ msg += ":"
555
+
556
+ msg += "\n"
557
+ for sub_err in exceptions:
558
+ msg += (
559
+ f"Jobscript {sub_err.js_idx} at path: {str(sub_err.js_path)!r}\n"
560
+ f"Submit command: {sub_err.submit_cmd!r}.\n"
561
+ f"Reason: {sub_err.message!r}\n"
562
+ )
563
+ if sub_err.subprocess_exc is not None:
564
+ msg += f"Subprocess exception: {sub_err.subprocess_exc}\n"
565
+ if sub_err.job_ID_parse_exc is not None:
566
+ msg += f"Subprocess job ID parse exception: {sub_err.job_ID_parse_exc}\n"
567
+ if sub_err.job_ID_parse_exc is not None:
568
+ msg += f"Job ID parse exception: {sub_err.job_ID_parse_exc}\n"
569
+ if sub_err.stdout:
570
+ msg += f"Submission stdout:\n{indent(sub_err.stdout, ' ')}\n"
571
+ if sub_err.stderr:
572
+ msg += f"Submission stderr:\n{indent(sub_err.stderr, ' ')}\n"
573
+ self.message = msg
574
+ super().__init__(msg)
329
575
 
330
576
 
331
577
  class WorkflowSubmissionFailure(RuntimeError):
@@ -333,6 +579,9 @@ class WorkflowSubmissionFailure(RuntimeError):
333
579
  A workflow submission failed.
334
580
  """
335
581
 
582
+ def __init__(self, exceptions: Sequence[SubmissionFailure]) -> None:
583
+ super().__init__("\n" + "\n\n".join(exn.message for exn in exceptions))
584
+
336
585
 
337
586
  class ResourceValidationError(ValueError):
338
587
  """An incompatible resource requested by the user."""
@@ -341,7 +590,7 @@ class ResourceValidationError(ValueError):
341
590
  class UnsupportedOSError(ResourceValidationError):
342
591
  """This machine is not of the requested OS."""
343
592
 
344
- def __init__(self, os_name) -> None:
593
+ def __init__(self, os_name: str) -> None:
345
594
  message = (
346
595
  f"OS {os_name!r} is not compatible with this machine/instance with OS: "
347
596
  f"{os.name!r}."
@@ -353,14 +602,15 @@ class UnsupportedOSError(ResourceValidationError):
353
602
  class UnsupportedShellError(ResourceValidationError):
354
603
  """We don't support this shell on this OS."""
355
604
 
356
- def __init__(self, shell, supported) -> None:
605
+ def __init__(self, shell: str, supported: Iterable[str]) -> None:
606
+ sup = set(supported)
357
607
  message = (
358
608
  f"Shell {shell!r} is not supported on this machine/instance. Supported "
359
- f"shells are: {supported!r}."
609
+ f"shells are: {sup!r}."
360
610
  )
361
611
  super().__init__(message)
362
612
  self.shell = shell
363
- self.supported = supported
613
+ self.supported = frozenset(sup)
364
614
 
365
615
 
366
616
  class UnsupportedSchedulerError(ResourceValidationError):
@@ -371,7 +621,12 @@ class UnsupportedSchedulerError(ResourceValidationError):
371
621
 
372
622
  """
373
623
 
374
- def __init__(self, scheduler, supported=None, available=None) -> None:
624
+ def __init__(
625
+ self,
626
+ scheduler: str,
627
+ supported: Iterable[str] | None = None,
628
+ available: Iterable[str] | None = None,
629
+ ) -> None:
375
630
  if supported is not None:
376
631
  message = (
377
632
  f"Scheduler {scheduler!r} is not supported on this machine/instance. "
@@ -385,8 +640,8 @@ class UnsupportedSchedulerError(ResourceValidationError):
385
640
  )
386
641
  super().__init__(message)
387
642
  self.scheduler = scheduler
388
- self.supported = supported
389
- self.available = available
643
+ self.supported = None if supported is None else tuple(supported)
644
+ self.available = None if available is None else tuple(available)
390
645
 
391
646
 
392
647
  class UnknownSGEPEError(ResourceValidationError):
@@ -394,42 +649,84 @@ class UnknownSGEPEError(ResourceValidationError):
394
649
  Miscellaneous error from SGE parallel environment.
395
650
  """
396
651
 
652
+ def __init__(self, env_name: str, all_env_names: Iterable[str]) -> None:
653
+ super().__init__(
654
+ f"The SGE parallel environment {env_name!r} is not "
655
+ f"specified in the configuration. Specified parallel environments "
656
+ f"are {sorted(all_env_names)!r}."
657
+ )
658
+
397
659
 
398
660
  class IncompatibleSGEPEError(ResourceValidationError):
399
661
  """
400
662
  The SGE parallel environment selected is incompatible.
401
663
  """
402
664
 
665
+ def __init__(self, env_name: str, num_cores: int | None) -> None:
666
+ super().__init__(
667
+ f"The SGE parallel environment {env_name!r} is not "
668
+ f"compatible with the number of cores requested: "
669
+ f"{num_cores!r}."
670
+ )
671
+
403
672
 
404
673
  class NoCompatibleSGEPEError(ResourceValidationError):
405
674
  """
406
675
  No SGE parallel environment is compatible with request.
407
676
  """
408
677
 
678
+ def __init__(self, num_cores: int | None) -> None:
679
+ super().__init__(
680
+ f"No compatible SGE parallel environment could be found for the "
681
+ f"specified `num_cores` ({num_cores!r})."
682
+ )
683
+
409
684
 
410
685
  class IncompatibleParallelModeError(ResourceValidationError):
411
686
  """
412
687
  The parallel mode is incompatible.
413
688
  """
414
689
 
690
+ def __init__(self, parallel_mode: ParallelMode) -> None:
691
+ super().__init__(
692
+ f"For the {parallel_mode.name.lower()} parallel mode, "
693
+ f"only a single node may be requested."
694
+ )
695
+
415
696
 
416
697
  class UnknownSLURMPartitionError(ResourceValidationError):
417
698
  """
418
699
  The requested SLURM partition isn't known.
419
700
  """
420
701
 
702
+ def __init__(self, part_name: str, all_parts: Iterable[str]) -> None:
703
+ super().__init__(
704
+ f"The SLURM partition {part_name!r} is not "
705
+ f"specified in the configuration. Specified partitions are "
706
+ f"{sorted(all_parts)!r}."
707
+ )
708
+
421
709
 
422
710
  class IncompatibleSLURMPartitionError(ResourceValidationError):
423
711
  """
424
712
  The requested SLURM partition is incompatible.
425
713
  """
426
714
 
715
+ def __init__(self, part_name: str, attr_kind: str, value) -> None:
716
+ super().__init__(
717
+ f"The SLURM partition {part_name!r} is not "
718
+ f"compatible with the {attr_kind} requested: {value!r}."
719
+ )
720
+
427
721
 
428
722
  class IncompatibleSLURMArgumentsError(ResourceValidationError):
429
723
  """
430
724
  The SLURM arguments are incompatible with each other.
431
725
  """
432
726
 
727
+ def __init__(self, msg: str) -> None:
728
+ super().__init__(msg)
729
+
433
730
 
434
731
  class _MissingStoreItemError(ValueError):
435
732
  def __init__(self, id_lst: Iterable[int], item_type: str) -> None:
@@ -485,23 +782,44 @@ class MissingParameterData(_MissingStoreItemError):
485
782
  super().__init__(id_lst, self._item_type)
486
783
 
487
784
 
785
+ class ParametersMetadataReadOnlyError(RuntimeError):
786
+ pass
787
+
788
+
488
789
  class NotSubmitMachineError(RuntimeError):
489
790
  """
490
791
  The requested machine can't be submitted to.
491
792
  """
492
793
 
794
+ def __init__(self) -> None:
795
+ super().__init__(
796
+ "Cannot get active state of the jobscript because the current machine "
797
+ "is not the machine on which the jobscript was submitted."
798
+ )
799
+
493
800
 
494
801
  class RunNotAbortableError(ValueError):
495
802
  """
496
803
  Cannot abort the run.
497
804
  """
498
805
 
806
+ def __init__(self) -> None:
807
+ super().__init__(
808
+ "The run is not defined as abortable in the task schema, so it cannot "
809
+ "be aborted."
810
+ )
811
+
499
812
 
500
813
  class NoCLIFormatMethodError(AttributeError):
501
814
  """
502
815
  Some CLI class lacks a format method
503
816
  """
504
817
 
818
+ def __init__(self, method: str, inp_val: object) -> None:
819
+ super().__init__(
820
+ f"No CLI format method {method!r} exists for the object {inp_val!r}."
821
+ )
822
+
505
823
 
506
824
  class ContainerKeyError(KeyError):
507
825
  """
@@ -509,11 +827,11 @@ class ContainerKeyError(KeyError):
509
827
 
510
828
  Parameters
511
829
  ----------
512
- path: list[str]
830
+ path:
513
831
  The path whose resolution failed.
514
832
  """
515
833
 
516
- def __init__(self, path: List[str]) -> None:
834
+ def __init__(self, path: list[str]) -> None:
517
835
  self.path = path
518
836
  super().__init__()
519
837
 
@@ -524,11 +842,11 @@ class MayNeedObjectError(Exception):
524
842
 
525
843
  Parameters
526
844
  ----------
527
- path: list[str]
845
+ path:
528
846
  The path whose resolution failed.
529
847
  """
530
848
 
531
- def __init__(self, path):
849
+ def __init__(self, path: str) -> None:
532
850
  self.path = path
533
851
  super().__init__()
534
852
 
@@ -538,12 +856,18 @@ class NoAvailableElementSetsError(Exception):
538
856
  No element set is available.
539
857
  """
540
858
 
859
+ def __init__(self) -> None:
860
+ super().__init__()
861
+
541
862
 
542
863
  class OutputFileParserNoOutputError(ValueError):
543
864
  """
544
865
  There was no output for the output file parser to parse.
545
866
  """
546
867
 
868
+ def __init__(self) -> None:
869
+ super().__init__()
870
+
547
871
 
548
872
  class SubmissionEnvironmentError(ValueError):
549
873
  """
@@ -551,67 +875,146 @@ class SubmissionEnvironmentError(ValueError):
551
875
  """
552
876
 
553
877
 
878
+ def _spec_to_ref(env_spec: Mapping[str, Any]):
879
+ non_name_spec = {k: v for k, v in env_spec.items() if k != "name"}
880
+ spec_str = f" with specifiers {non_name_spec!r}" if non_name_spec else ""
881
+ return f"{env_spec['name']!r}{spec_str}"
882
+
883
+
554
884
  class MissingEnvironmentExecutableError(SubmissionEnvironmentError):
555
885
  """
556
886
  The environment does not have the requested executable at all.
557
887
  """
558
888
 
889
+ def __init__(self, env_spec: Mapping[str, Any], exec_label: str) -> None:
890
+ super().__init__(
891
+ f"The environment {_spec_to_ref(env_spec)} as defined on this machine has no "
892
+ f"executable labelled {exec_label!r}, which is required for this "
893
+ f"submission, so the submission cannot be created."
894
+ )
895
+
559
896
 
560
897
  class MissingEnvironmentExecutableInstanceError(SubmissionEnvironmentError):
561
898
  """
562
899
  The environment does not have a suitable instance of the requested executable.
563
900
  """
564
901
 
902
+ def __init__(
903
+ self, env_spec: Mapping[str, Any], exec_label: str, js_idx: int, res: dict
904
+ ) -> None:
905
+ super().__init__(
906
+ f"No matching executable instances found for executable "
907
+ f"{exec_label!r} of environment {_spec_to_ref(env_spec)} for jobscript "
908
+ f"index {js_idx!r} with requested resources {res!r}."
909
+ )
910
+
565
911
 
566
912
  class MissingEnvironmentError(SubmissionEnvironmentError):
567
913
  """
568
914
  There is no environment with that name.
569
915
  """
570
916
 
917
+ def __init__(self, env_spec: Mapping[str, Any]) -> None:
918
+ super().__init__(
919
+ f"The environment {_spec_to_ref(env_spec)} is not defined on this machine, so the "
920
+ f"submission cannot be created."
921
+ )
922
+
571
923
 
572
924
  class UnsupportedScriptDataFormat(ValueError):
573
925
  """
574
926
  That format of script data is not supported.
575
927
  """
576
928
 
929
+ def __init__(
930
+ self, data: ScriptData, kind: str, name: str, formats: tuple[str, ...]
931
+ ) -> None:
932
+ super().__init__(
933
+ f"Script data format {data!r} for {kind} parameter {name!r} is not "
934
+ f"understood. Available script data formats are: "
935
+ f"{formats!r}."
936
+ )
937
+
577
938
 
578
939
  class UnknownScriptDataParameter(ValueError):
579
940
  """
580
941
  Unknown parameter in script data.
581
942
  """
582
943
 
944
+ def __init__(self, name: str, kind: str, param_names: Sequence[str]) -> None:
945
+ super().__init__(
946
+ f"Script data parameter {name!r} is not a known parameter of the "
947
+ f"action. Parameters ({kind}) are: {param_names!r}."
948
+ )
949
+
583
950
 
584
951
  class UnknownScriptDataKey(ValueError):
585
952
  """
586
953
  Unknown key in script data.
587
954
  """
588
955
 
956
+ def __init__(self, key: str, allowed_keys: Sequence[str]) -> None:
957
+ super().__init__(
958
+ f"Script data key {key!r} is not understood. Allowed keys are: "
959
+ f"{allowed_keys!r}."
960
+ )
961
+
589
962
 
590
963
  class MissingVariableSubstitutionError(KeyError):
591
964
  """
592
965
  No definition available of a variable being substituted.
593
966
  """
594
967
 
968
+ def __init__(self, var_name: str, variables: Iterable[str]) -> None:
969
+ super().__init__(
970
+ f"The variable {var_name!r} referenced in the string does not match "
971
+ f"any of the provided variables: {sorted(variables)!r}."
972
+ )
973
+
595
974
 
596
975
  class EnvironmentPresetUnknownEnvironmentError(ValueError):
597
976
  """
598
977
  An environment preset could not be resolved to an execution environment.
599
978
  """
600
979
 
980
+ def __init__(self, name: str, bad_envs: Iterable[str]) -> None:
981
+ super().__init__(
982
+ f"Task schema {name} has environment presets that refer to one "
983
+ f"or more environments that are not referenced in any of the task "
984
+ f"schema's actions: {', '.join(f'{env!r}' for env in sorted(bad_envs))}."
985
+ )
986
+
601
987
 
602
988
  class UnknownEnvironmentPresetError(ValueError):
603
989
  """
604
990
  An execution environment was unknown.
605
991
  """
606
992
 
993
+ def __init__(self, preset_name: str, schema_name: str) -> None:
994
+ super().__init__(
995
+ f"There is no environment preset named {preset_name!r} defined "
996
+ f"in the task schema {schema_name}."
997
+ )
998
+
607
999
 
608
1000
  class MultipleEnvironmentsError(ValueError):
609
1001
  """
610
1002
  Multiple applicable execution environments exist.
611
1003
  """
612
1004
 
1005
+ def __init__(self, env_spec: Mapping[str, Any]) -> None:
1006
+ super().__init__(
1007
+ f"Multiple environments {_spec_to_ref(env_spec)} are defined on this machine."
1008
+ )
1009
+
613
1010
 
614
1011
  class MissingElementGroup(ValueError):
615
1012
  """
616
1013
  An element group should exist but doesn't.
617
1014
  """
1015
+
1016
+ def __init__(self, task_name: str, group_name: str, input_path: str) -> None:
1017
+ super().__init__(
1018
+ f"Adding elements to task {task_name!r}: "
1019
+ f"no element group named {group_name!r} found for input {input_path!r}."
1020
+ )