hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a190__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/__pyinstaller/hook-hpcflow.py +8 -6
- hpcflow/_version.py +1 -1
- hpcflow/app.py +1 -0
- hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
- hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
- hpcflow/sdk/__init__.py +21 -15
- hpcflow/sdk/app.py +2133 -770
- hpcflow/sdk/cli.py +281 -250
- hpcflow/sdk/cli_common.py +6 -2
- hpcflow/sdk/config/__init__.py +1 -1
- hpcflow/sdk/config/callbacks.py +77 -42
- hpcflow/sdk/config/cli.py +126 -103
- hpcflow/sdk/config/config.py +578 -311
- hpcflow/sdk/config/config_file.py +131 -95
- hpcflow/sdk/config/errors.py +112 -85
- hpcflow/sdk/config/types.py +145 -0
- hpcflow/sdk/core/actions.py +1054 -994
- hpcflow/sdk/core/app_aware.py +24 -0
- hpcflow/sdk/core/cache.py +81 -63
- hpcflow/sdk/core/command_files.py +275 -185
- hpcflow/sdk/core/commands.py +111 -107
- hpcflow/sdk/core/element.py +724 -503
- hpcflow/sdk/core/enums.py +192 -0
- hpcflow/sdk/core/environment.py +74 -93
- hpcflow/sdk/core/errors.py +398 -51
- hpcflow/sdk/core/json_like.py +540 -272
- hpcflow/sdk/core/loop.py +380 -334
- hpcflow/sdk/core/loop_cache.py +160 -43
- hpcflow/sdk/core/object_list.py +370 -207
- hpcflow/sdk/core/parameters.py +728 -600
- hpcflow/sdk/core/rule.py +59 -41
- hpcflow/sdk/core/run_dir_files.py +33 -22
- hpcflow/sdk/core/task.py +1546 -1325
- hpcflow/sdk/core/task_schema.py +240 -196
- hpcflow/sdk/core/test_utils.py +126 -88
- hpcflow/sdk/core/types.py +387 -0
- hpcflow/sdk/core/utils.py +410 -305
- hpcflow/sdk/core/validation.py +82 -9
- hpcflow/sdk/core/workflow.py +1192 -1028
- hpcflow/sdk/core/zarr_io.py +98 -137
- hpcflow/sdk/demo/cli.py +46 -33
- hpcflow/sdk/helper/cli.py +18 -16
- hpcflow/sdk/helper/helper.py +75 -63
- hpcflow/sdk/helper/watcher.py +61 -28
- hpcflow/sdk/log.py +83 -59
- hpcflow/sdk/persistence/__init__.py +8 -31
- hpcflow/sdk/persistence/base.py +988 -586
- hpcflow/sdk/persistence/defaults.py +6 -0
- hpcflow/sdk/persistence/discovery.py +38 -0
- hpcflow/sdk/persistence/json.py +408 -153
- hpcflow/sdk/persistence/pending.py +158 -123
- hpcflow/sdk/persistence/store_resource.py +37 -22
- hpcflow/sdk/persistence/types.py +307 -0
- hpcflow/sdk/persistence/utils.py +14 -11
- hpcflow/sdk/persistence/zarr.py +477 -420
- hpcflow/sdk/runtime.py +44 -41
- hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
- hpcflow/sdk/submission/jobscript.py +444 -404
- hpcflow/sdk/submission/schedulers/__init__.py +133 -40
- hpcflow/sdk/submission/schedulers/direct.py +97 -71
- hpcflow/sdk/submission/schedulers/sge.py +132 -126
- hpcflow/sdk/submission/schedulers/slurm.py +263 -268
- hpcflow/sdk/submission/schedulers/utils.py +7 -2
- hpcflow/sdk/submission/shells/__init__.py +14 -15
- hpcflow/sdk/submission/shells/base.py +102 -29
- hpcflow/sdk/submission/shells/bash.py +72 -55
- hpcflow/sdk/submission/shells/os_version.py +31 -30
- hpcflow/sdk/submission/shells/powershell.py +37 -29
- hpcflow/sdk/submission/submission.py +203 -257
- hpcflow/sdk/submission/types.py +143 -0
- hpcflow/sdk/typing.py +163 -12
- hpcflow/tests/conftest.py +8 -6
- hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
- hpcflow/tests/scripts/test_main_scripts.py +60 -30
- hpcflow/tests/shells/wsl/test_wsl_submission.py +6 -4
- hpcflow/tests/unit/test_action.py +86 -75
- hpcflow/tests/unit/test_action_rule.py +9 -4
- hpcflow/tests/unit/test_app.py +13 -6
- hpcflow/tests/unit/test_cli.py +1 -1
- hpcflow/tests/unit/test_command.py +71 -54
- hpcflow/tests/unit/test_config.py +20 -15
- hpcflow/tests/unit/test_config_file.py +21 -18
- hpcflow/tests/unit/test_element.py +58 -62
- hpcflow/tests/unit/test_element_iteration.py +3 -1
- hpcflow/tests/unit/test_element_set.py +29 -19
- hpcflow/tests/unit/test_group.py +4 -2
- hpcflow/tests/unit/test_input_source.py +116 -93
- hpcflow/tests/unit/test_input_value.py +29 -24
- hpcflow/tests/unit/test_json_like.py +44 -35
- hpcflow/tests/unit/test_loop.py +65 -58
- hpcflow/tests/unit/test_object_list.py +17 -12
- hpcflow/tests/unit/test_parameter.py +16 -7
- hpcflow/tests/unit/test_persistence.py +48 -35
- hpcflow/tests/unit/test_resources.py +20 -18
- hpcflow/tests/unit/test_run.py +8 -3
- hpcflow/tests/unit/test_runtime.py +2 -1
- hpcflow/tests/unit/test_schema_input.py +23 -15
- hpcflow/tests/unit/test_shell.py +3 -2
- hpcflow/tests/unit/test_slurm.py +8 -7
- hpcflow/tests/unit/test_submission.py +39 -19
- hpcflow/tests/unit/test_task.py +352 -247
- hpcflow/tests/unit/test_task_schema.py +33 -20
- hpcflow/tests/unit/test_utils.py +9 -11
- hpcflow/tests/unit/test_value_sequence.py +15 -12
- hpcflow/tests/unit/test_workflow.py +114 -83
- hpcflow/tests/unit/test_workflow_template.py +0 -1
- hpcflow/tests/workflows/test_jobscript.py +2 -1
- hpcflow/tests/workflows/test_workflows.py +18 -13
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/METADATA +2 -1
- hpcflow_new2-0.2.0a190.dist-info/RECORD +165 -0
- hpcflow/sdk/core/parallel.py +0 -21
- hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/core/errors.py
CHANGED
@@ -2,8 +2,17 @@
|
|
2
2
|
Errors from the workflow system.
|
3
3
|
"""
|
4
4
|
|
5
|
+
from __future__ import annotations
|
5
6
|
import os
|
6
|
-
from
|
7
|
+
from collections.abc import Iterable, Mapping, Sequence
|
8
|
+
from textwrap import indent
|
9
|
+
from typing import Any, TYPE_CHECKING
|
10
|
+
|
11
|
+
if TYPE_CHECKING:
|
12
|
+
from .enums import ParallelMode
|
13
|
+
from .object_list import WorkflowLoopList
|
14
|
+
from .parameters import InputSource, ValueSequence
|
15
|
+
from .types import ScriptData
|
7
16
|
|
8
17
|
|
9
18
|
class InputValueDuplicateSequenceAddress(ValueError):
|
@@ -11,36 +20,65 @@ class InputValueDuplicateSequenceAddress(ValueError):
|
|
11
20
|
An InputValue has the same sequence address twice.
|
12
21
|
"""
|
13
22
|
|
23
|
+
# FIXME: never used
|
24
|
+
|
14
25
|
|
15
26
|
class TaskTemplateMultipleSchemaObjectives(ValueError):
|
16
27
|
"""
|
17
28
|
A TaskTemplate has multiple objectives.
|
18
29
|
"""
|
19
30
|
|
31
|
+
def __init__(self, names: set[str]) -> None:
|
32
|
+
super().__init__(
|
33
|
+
f"All task schemas used within a task must have the same "
|
34
|
+
f"objective, but found multiple objectives: {sorted(names)!r}"
|
35
|
+
)
|
36
|
+
|
20
37
|
|
21
38
|
class TaskTemplateUnexpectedInput(ValueError):
|
22
39
|
"""
|
23
40
|
A TaskTemplate was given unexpected input.
|
24
41
|
"""
|
25
42
|
|
43
|
+
def __init__(self, unexpected_types: set[str]) -> None:
|
44
|
+
super().__init__(
|
45
|
+
f"The following input parameters are unexpected: {sorted(unexpected_types)!r}"
|
46
|
+
)
|
47
|
+
|
26
48
|
|
27
49
|
class TaskTemplateUnexpectedSequenceInput(ValueError):
|
28
50
|
"""
|
29
51
|
A TaskTemplate was given an unexpected sequence.
|
30
52
|
"""
|
31
53
|
|
54
|
+
def __init__(
|
55
|
+
self, inp_type: str, expected_types: set[str], seq: ValueSequence
|
56
|
+
) -> None:
|
57
|
+
allowed_str = ", ".join(f'"{in_typ}"' for in_typ in expected_types)
|
58
|
+
super().__init__(
|
59
|
+
f"The input type {inp_type!r} specified in the following sequence"
|
60
|
+
f" path is unexpected: {seq.path!r}. Available input types are: "
|
61
|
+
f"{allowed_str}."
|
62
|
+
)
|
63
|
+
|
32
64
|
|
33
65
|
class TaskTemplateMultipleInputValues(ValueError):
|
34
66
|
"""
|
35
67
|
A TaskTemplate had multiple input values bound over each other.
|
36
68
|
"""
|
37
69
|
|
70
|
+
def __init__(self, msg: str) -> None:
|
71
|
+
super().__init__(msg)
|
72
|
+
|
38
73
|
|
39
74
|
class InvalidIdentifier(ValueError):
|
40
75
|
"""
|
41
76
|
A bad identifier name was given.
|
42
77
|
"""
|
43
78
|
|
79
|
+
def __init__(self, name: str) -> None:
|
80
|
+
super().__init__(f"Invalid string for identifier: {name!r}")
|
81
|
+
|
44
82
|
|
45
83
|
class MissingInputs(Exception):
|
46
84
|
"""
|
@@ -48,17 +86,16 @@ class MissingInputs(Exception):
|
|
48
86
|
|
49
87
|
Parameters
|
50
88
|
----------
|
51
|
-
|
52
|
-
The message of the exception.
|
53
|
-
missing_inputs: list[str]
|
89
|
+
missing_inputs:
|
54
90
|
The missing inputs.
|
55
91
|
"""
|
56
92
|
|
57
93
|
# TODO: add links to doc pages for common user-exceptions?
|
58
94
|
|
59
|
-
def __init__(self,
|
60
|
-
self.missing_inputs = missing_inputs
|
61
|
-
|
95
|
+
def __init__(self, missing_inputs: Iterable[str]) -> None:
|
96
|
+
self.missing_inputs = tuple(missing_inputs)
|
97
|
+
missing_str = ", ".join(map(repr, missing_inputs))
|
98
|
+
super().__init__(f"The following inputs have no sources: {missing_str}.")
|
62
99
|
|
63
100
|
|
64
101
|
class UnrequiredInputSources(ValueError):
|
@@ -67,23 +104,23 @@ class UnrequiredInputSources(ValueError):
|
|
67
104
|
|
68
105
|
Parameters
|
69
106
|
----------
|
70
|
-
|
71
|
-
The message of the exception.
|
72
|
-
unrequired_sources: list
|
107
|
+
unrequired_sources:
|
73
108
|
The input sources that were not required.
|
74
109
|
"""
|
75
110
|
|
76
|
-
def __init__(self,
|
77
|
-
self.unrequired_sources = unrequired_sources
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
111
|
+
def __init__(self, unrequired_sources: Iterable[str]) -> None:
|
112
|
+
self.unrequired_sources = frozenset(unrequired_sources)
|
113
|
+
message = (
|
114
|
+
f"The following input sources are not required but have been specified: "
|
115
|
+
f'{", ".join(map(repr, sorted(self.unrequired_sources)))}.'
|
116
|
+
)
|
117
|
+
if any((bad := src).startswith("inputs.") for src in self.unrequired_sources):
|
118
|
+
# reminder about how to specify input sources:
|
119
|
+
message += (
|
120
|
+
f" Note that input source keys should not be specified with the "
|
121
|
+
f"'inputs.' prefix. Did you mean to specify "
|
122
|
+
f"{bad.removeprefix('inputs.')!r} instead of {bad!r}?"
|
123
|
+
)
|
87
124
|
super().__init__(message)
|
88
125
|
|
89
126
|
|
@@ -93,15 +130,16 @@ class ExtraInputs(Exception):
|
|
93
130
|
|
94
131
|
Parameters
|
95
132
|
----------
|
96
|
-
|
97
|
-
The message of the exception.
|
98
|
-
extra_inputs: list
|
133
|
+
extra_inputs:
|
99
134
|
The extra inputs.
|
100
135
|
"""
|
101
136
|
|
102
|
-
def __init__(self,
|
103
|
-
self.extra_inputs = extra_inputs
|
104
|
-
super().__init__(
|
137
|
+
def __init__(self, extra_inputs: set[str]) -> None:
|
138
|
+
self.extra_inputs = frozenset(extra_inputs)
|
139
|
+
super().__init__(
|
140
|
+
f"The following inputs are not required, but have been passed: "
|
141
|
+
f'{", ".join(f"{typ!r}" for typ in extra_inputs)}.'
|
142
|
+
)
|
105
143
|
|
106
144
|
|
107
145
|
class UnavailableInputSource(ValueError):
|
@@ -109,184 +147,311 @@ class UnavailableInputSource(ValueError):
|
|
109
147
|
An input source was not available.
|
110
148
|
"""
|
111
149
|
|
150
|
+
def __init__(
|
151
|
+
self, source: InputSource, path: str, avail: Sequence[InputSource]
|
152
|
+
) -> None:
|
153
|
+
super().__init__(
|
154
|
+
f"The input source {source.to_string()!r} is not "
|
155
|
+
f"available for input path {path!r}. Available "
|
156
|
+
f"input sources are: {[src.to_string() for src in avail]}."
|
157
|
+
)
|
158
|
+
|
112
159
|
|
113
160
|
class InapplicableInputSourceElementIters(ValueError):
|
114
161
|
"""
|
115
162
|
An input source element iteration was inapplicable."""
|
116
163
|
|
164
|
+
def __init__(self, source: InputSource, elem_iters_IDs: Sequence[int] | None) -> None:
|
165
|
+
super().__init__(
|
166
|
+
f"The specified `element_iters` for input source "
|
167
|
+
f"{source.to_string()!r} are not all applicable. "
|
168
|
+
f"Applicable element iteration IDs for this input source "
|
169
|
+
f"are: {elem_iters_IDs!r}."
|
170
|
+
)
|
171
|
+
|
117
172
|
|
118
173
|
class NoCoincidentInputSources(ValueError):
|
119
174
|
"""
|
120
175
|
Could not line up input sources to make an actual valid execution.
|
121
176
|
"""
|
122
177
|
|
178
|
+
def __init__(self, name: str, task_ref: int) -> None:
|
179
|
+
super().__init__(
|
180
|
+
f"Task {name!r}: input sources from task {task_ref!r} have "
|
181
|
+
f"no coincident applicable element iterations. Consider setting "
|
182
|
+
f"the element set (or task) argument "
|
183
|
+
f"`allow_non_coincident_task_sources` to `True`, which will "
|
184
|
+
f"allow for input sources from the same task to use different "
|
185
|
+
f"(non-coinciding) subsets of element iterations from the "
|
186
|
+
f"source task."
|
187
|
+
)
|
188
|
+
|
123
189
|
|
124
190
|
class TaskTemplateInvalidNesting(ValueError):
|
125
191
|
"""
|
126
192
|
Invalid nesting in a task template.
|
127
193
|
"""
|
128
194
|
|
195
|
+
def __init__(self, key: str, value: float) -> None:
|
196
|
+
super().__init__(
|
197
|
+
f"`nesting_order` must be >=0 for all keys, but for key {key!r}, value "
|
198
|
+
f"of {value!r} was specified."
|
199
|
+
)
|
200
|
+
|
129
201
|
|
130
202
|
class TaskSchemaSpecValidationError(Exception):
|
131
203
|
"""
|
132
204
|
A task schema failed to validate.
|
133
205
|
"""
|
134
206
|
|
207
|
+
# FIXME: never used
|
208
|
+
|
135
209
|
|
136
210
|
class WorkflowSpecValidationError(Exception):
|
137
211
|
"""
|
138
212
|
A workflow failed to validate.
|
139
213
|
"""
|
140
214
|
|
215
|
+
# FIXME: never used
|
216
|
+
|
141
217
|
|
142
218
|
class InputSourceValidationError(Exception):
|
143
219
|
"""
|
144
220
|
An input source failed to validate.
|
145
221
|
"""
|
146
222
|
|
223
|
+
# FIXME: never used
|
224
|
+
|
147
225
|
|
148
226
|
class EnvironmentSpecValidationError(Exception):
|
149
227
|
"""
|
150
228
|
An environment specification failed to validate.
|
151
229
|
"""
|
152
230
|
|
231
|
+
# FIXME: never used
|
232
|
+
|
153
233
|
|
154
234
|
class ParameterSpecValidationError(Exception):
|
155
235
|
"""
|
156
236
|
A parameter specification failed to validate.
|
157
237
|
"""
|
158
238
|
|
239
|
+
# FIXME: never used
|
240
|
+
|
159
241
|
|
160
242
|
class FileSpecValidationError(Exception):
|
161
243
|
"""
|
162
244
|
A file specification failed to validate.
|
163
245
|
"""
|
164
246
|
|
247
|
+
# FIXME: never used
|
248
|
+
|
165
249
|
|
166
250
|
class DuplicateExecutableError(ValueError):
|
167
251
|
"""
|
168
252
|
The same executable was present twice in an executable environment.
|
169
253
|
"""
|
170
254
|
|
255
|
+
def __init__(self, duplicate_labels: list) -> None:
|
256
|
+
super().__init__(
|
257
|
+
f"Executables must have unique `label`s within each environment, but "
|
258
|
+
f"found label(s) multiple times: {duplicate_labels!r}"
|
259
|
+
)
|
260
|
+
|
171
261
|
|
172
262
|
class MissingCompatibleActionEnvironment(Exception):
|
173
263
|
"""
|
174
264
|
Could not find a compatible action environment.
|
175
265
|
"""
|
176
266
|
|
267
|
+
def __init__(self, msg: str) -> None:
|
268
|
+
super().__init__(f"No compatible environment is specified for the {msg}.")
|
269
|
+
|
177
270
|
|
178
271
|
class MissingActionEnvironment(Exception):
|
179
272
|
"""
|
180
273
|
Could not find an action environment.
|
181
274
|
"""
|
182
275
|
|
276
|
+
# FIXME: never used
|
277
|
+
|
183
278
|
|
184
279
|
class ActionEnvironmentMissingNameError(Exception):
|
185
280
|
"""
|
186
281
|
An action environment was missing its name.
|
187
282
|
"""
|
188
283
|
|
284
|
+
def __init__(self, environment: Mapping[str, Any]) -> None:
|
285
|
+
super().__init__(
|
286
|
+
"The action-environment environment specification must include a string "
|
287
|
+
"`name` key, or be specified as string that is that name. Provided "
|
288
|
+
f"environment key was {environment!r}."
|
289
|
+
)
|
290
|
+
|
189
291
|
|
190
292
|
class FromSpecMissingObjectError(Exception):
|
191
293
|
"""
|
192
294
|
Missing object when deserialising from specification.
|
193
295
|
"""
|
194
296
|
|
297
|
+
# FIXME: never used
|
298
|
+
|
195
299
|
|
196
300
|
class TaskSchemaMissingParameterError(Exception):
|
197
301
|
"""
|
198
302
|
Parameter was missing from task schema.
|
199
303
|
"""
|
200
304
|
|
305
|
+
# FIXME: never used
|
306
|
+
|
201
307
|
|
202
308
|
class ToJSONLikeChildReferenceError(Exception):
|
203
309
|
"""
|
204
310
|
Failed to generate or reference a child object when converting to JSON.
|
205
311
|
"""
|
206
312
|
|
313
|
+
# FIXME: never thrown
|
314
|
+
|
207
315
|
|
208
316
|
class InvalidInputSourceTaskReference(Exception):
|
209
317
|
"""
|
210
318
|
Invalid input source in task reference.
|
211
319
|
"""
|
212
320
|
|
321
|
+
def __init__(self, input_source: InputSource, task_ref: int | None = None) -> None:
|
322
|
+
super().__init__(
|
323
|
+
f"Input source {input_source.to_string()!r} cannot refer to the "
|
324
|
+
f"outputs of its own task!"
|
325
|
+
if task_ref is None
|
326
|
+
else f"Input source {input_source.to_string()!r} refers to a missing "
|
327
|
+
f"or inaccessible task: {task_ref!r}."
|
328
|
+
)
|
329
|
+
|
213
330
|
|
214
331
|
class WorkflowNotFoundError(Exception):
|
215
332
|
"""
|
216
333
|
Could not find the workflow.
|
217
334
|
"""
|
218
335
|
|
336
|
+
def __init__(self, path, fs) -> None:
|
337
|
+
super().__init__(
|
338
|
+
f"Cannot infer a store format at path {path!r} with file system {fs!r}."
|
339
|
+
)
|
340
|
+
|
219
341
|
|
220
342
|
class MalformedWorkflowError(Exception):
|
221
343
|
"""
|
222
344
|
Workflow was a malformed document.
|
223
345
|
"""
|
224
346
|
|
347
|
+
# TODO: use this class
|
348
|
+
|
225
349
|
|
226
350
|
class ValuesAlreadyPersistentError(Exception):
|
227
351
|
"""
|
228
352
|
Trying to make a value persistent that already is so.
|
229
353
|
"""
|
230
354
|
|
355
|
+
# FIXME: never used
|
356
|
+
|
231
357
|
|
232
358
|
class MalformedParameterPathError(ValueError):
|
233
359
|
"""
|
234
360
|
The path to a parameter was ill-formed.
|
235
361
|
"""
|
236
362
|
|
363
|
+
def __init__(self, msg: str) -> None:
|
364
|
+
super().__init__(msg)
|
365
|
+
|
237
366
|
|
238
367
|
class MalformedNestingOrderPath(ValueError):
|
239
368
|
"""
|
240
369
|
A nesting order path was ill-formed.
|
241
370
|
"""
|
242
371
|
|
372
|
+
def __init__(self, k: str, allowed_nesting_paths: Iterable[str]) -> None:
|
373
|
+
super().__init__(
|
374
|
+
f"Element set: nesting order path {k!r} not understood. Each key in "
|
375
|
+
f"`nesting_order` must be start with one of "
|
376
|
+
f"{sorted(allowed_nesting_paths)!r}."
|
377
|
+
)
|
378
|
+
|
243
379
|
|
244
380
|
class UnknownResourceSpecItemError(ValueError):
|
245
381
|
"""
|
246
382
|
A resource specification item was not found.
|
247
383
|
"""
|
248
384
|
|
385
|
+
def __init__(self, msg: str) -> None:
|
386
|
+
super().__init__(msg)
|
387
|
+
|
249
388
|
|
250
389
|
class WorkflowParameterMissingError(AttributeError):
|
251
390
|
"""
|
252
391
|
A parameter to a workflow was missing.
|
253
392
|
"""
|
254
393
|
|
394
|
+
# FIXME: never thrown
|
395
|
+
|
255
396
|
|
256
397
|
class WorkflowBatchUpdateFailedError(Exception):
|
257
398
|
"""
|
258
399
|
An update to a workflow failed.
|
259
400
|
"""
|
260
401
|
|
402
|
+
# FIXME: only throw is commented out?
|
403
|
+
|
261
404
|
|
262
405
|
class WorkflowLimitsError(ValueError):
|
263
406
|
"""
|
264
407
|
Workflow hit limits.
|
265
408
|
"""
|
266
409
|
|
410
|
+
# FIXME: never used
|
411
|
+
|
267
412
|
|
268
413
|
class UnsetParameterDataError(Exception):
|
269
414
|
"""
|
270
415
|
Tried to read from an unset parameter.
|
271
416
|
"""
|
272
417
|
|
418
|
+
def __init__(self, path: str | None, path_i: str) -> None:
|
419
|
+
super().__init__(
|
420
|
+
f"Element data path {path!r} resolves to unset data for "
|
421
|
+
f"(at least) data-index path: {path_i!r}."
|
422
|
+
)
|
423
|
+
|
273
424
|
|
274
425
|
class LoopAlreadyExistsError(Exception):
|
275
426
|
"""
|
276
427
|
A particular loop (or its name) already exists.
|
277
428
|
"""
|
278
429
|
|
430
|
+
def __init__(self, loop_name: str, loops: WorkflowLoopList) -> None:
|
431
|
+
super().__init__(
|
432
|
+
f"A loop with the name {loop_name!r} already exists in the workflow: "
|
433
|
+
f"{getattr(loops, loop_name)!r}."
|
434
|
+
)
|
435
|
+
|
279
436
|
|
280
437
|
class LoopTaskSubsetError(ValueError):
|
281
438
|
"""
|
282
439
|
Problem constructing a subset of a task for a loop.
|
283
440
|
"""
|
284
441
|
|
442
|
+
def __init__(self, loop_name: str, task_indices: Sequence[int]) -> None:
|
443
|
+
super().__init__(
|
444
|
+
f"Loop {loop_name!r}: task subset must be an ascending contiguous range, "
|
445
|
+
f"but specified task indices were: {task_indices!r}."
|
446
|
+
)
|
447
|
+
|
285
448
|
|
286
449
|
class SchedulerVersionsFailure(RuntimeError):
|
287
450
|
"""We couldn't get the scheduler and or shell versions."""
|
288
451
|
|
289
|
-
|
452
|
+
# FIXME: unused
|
453
|
+
|
454
|
+
def __init__(self, message: str) -> None:
|
290
455
|
self.message = message
|
291
456
|
super().__init__(message)
|
292
457
|
|
@@ -298,15 +463,16 @@ class JobscriptSubmissionFailure(RuntimeError):
|
|
298
463
|
|
299
464
|
def __init__(
|
300
465
|
self,
|
301
|
-
message,
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
466
|
+
message: str,
|
467
|
+
*,
|
468
|
+
submit_cmd: list[str],
|
469
|
+
js_idx: int,
|
470
|
+
js_path: str,
|
471
|
+
stdout: str | None = None,
|
472
|
+
stderr: str | None = None,
|
473
|
+
subprocess_exc: Exception | None = None,
|
474
|
+
job_ID_parse_exc: Exception | None = None,
|
475
|
+
):
|
310
476
|
self.message = message
|
311
477
|
self.submit_cmd = submit_cmd
|
312
478
|
self.js_idx = js_idx
|
@@ -323,9 +489,37 @@ class SubmissionFailure(RuntimeError):
|
|
323
489
|
A job submission failed.
|
324
490
|
"""
|
325
491
|
|
326
|
-
def __init__(
|
327
|
-
self
|
328
|
-
|
492
|
+
def __init__(
|
493
|
+
self,
|
494
|
+
sub_idx: int,
|
495
|
+
submitted_js_idx: Sequence[int],
|
496
|
+
exceptions: Iterable[JobscriptSubmissionFailure],
|
497
|
+
) -> None:
|
498
|
+
msg = f"Some jobscripts in submission index {sub_idx} could not be submitted"
|
499
|
+
if submitted_js_idx:
|
500
|
+
msg += f" (but jobscripts {submitted_js_idx} were submitted successfully):"
|
501
|
+
else:
|
502
|
+
msg += ":"
|
503
|
+
|
504
|
+
msg += "\n"
|
505
|
+
for sub_err in exceptions:
|
506
|
+
msg += (
|
507
|
+
f"Jobscript {sub_err.js_idx} at path: {str(sub_err.js_path)!r}\n"
|
508
|
+
f"Submit command: {sub_err.submit_cmd!r}.\n"
|
509
|
+
f"Reason: {sub_err.message!r}\n"
|
510
|
+
)
|
511
|
+
if sub_err.subprocess_exc is not None:
|
512
|
+
msg += f"Subprocess exception: {sub_err.subprocess_exc}\n"
|
513
|
+
if sub_err.job_ID_parse_exc is not None:
|
514
|
+
msg += f"Subprocess job ID parse exception: {sub_err.job_ID_parse_exc}\n"
|
515
|
+
if sub_err.job_ID_parse_exc is not None:
|
516
|
+
msg += f"Job ID parse exception: {sub_err.job_ID_parse_exc}\n"
|
517
|
+
if sub_err.stdout:
|
518
|
+
msg += f"Submission stdout:\n{indent(sub_err.stdout, ' ')}\n"
|
519
|
+
if sub_err.stderr:
|
520
|
+
msg += f"Submission stderr:\n{indent(sub_err.stderr, ' ')}\n"
|
521
|
+
self.message = msg
|
522
|
+
super().__init__(msg)
|
329
523
|
|
330
524
|
|
331
525
|
class WorkflowSubmissionFailure(RuntimeError):
|
@@ -333,6 +527,9 @@ class WorkflowSubmissionFailure(RuntimeError):
|
|
333
527
|
A workflow submission failed.
|
334
528
|
"""
|
335
529
|
|
530
|
+
def __init__(self, exceptions: Sequence[SubmissionFailure]) -> None:
|
531
|
+
super().__init__("\n" + "\n\n".join(exn.message for exn in exceptions))
|
532
|
+
|
336
533
|
|
337
534
|
class ResourceValidationError(ValueError):
|
338
535
|
"""An incompatible resource requested by the user."""
|
@@ -341,7 +538,7 @@ class ResourceValidationError(ValueError):
|
|
341
538
|
class UnsupportedOSError(ResourceValidationError):
|
342
539
|
"""This machine is not of the requested OS."""
|
343
540
|
|
344
|
-
def __init__(self, os_name) -> None:
|
541
|
+
def __init__(self, os_name: str) -> None:
|
345
542
|
message = (
|
346
543
|
f"OS {os_name!r} is not compatible with this machine/instance with OS: "
|
347
544
|
f"{os.name!r}."
|
@@ -353,14 +550,15 @@ class UnsupportedOSError(ResourceValidationError):
|
|
353
550
|
class UnsupportedShellError(ResourceValidationError):
|
354
551
|
"""We don't support this shell on this OS."""
|
355
552
|
|
356
|
-
def __init__(self, shell, supported) -> None:
|
553
|
+
def __init__(self, shell: str, supported: Iterable[str]) -> None:
|
554
|
+
sup = set(supported)
|
357
555
|
message = (
|
358
556
|
f"Shell {shell!r} is not supported on this machine/instance. Supported "
|
359
|
-
f"shells are: {
|
557
|
+
f"shells are: {sup!r}."
|
360
558
|
)
|
361
559
|
super().__init__(message)
|
362
560
|
self.shell = shell
|
363
|
-
self.supported =
|
561
|
+
self.supported = frozenset(sup)
|
364
562
|
|
365
563
|
|
366
564
|
class UnsupportedSchedulerError(ResourceValidationError):
|
@@ -371,7 +569,12 @@ class UnsupportedSchedulerError(ResourceValidationError):
|
|
371
569
|
|
372
570
|
"""
|
373
571
|
|
374
|
-
def __init__(
|
572
|
+
def __init__(
|
573
|
+
self,
|
574
|
+
scheduler: str,
|
575
|
+
supported: Iterable[str] | None = None,
|
576
|
+
available: Iterable[str] | None = None,
|
577
|
+
) -> None:
|
375
578
|
if supported is not None:
|
376
579
|
message = (
|
377
580
|
f"Scheduler {scheduler!r} is not supported on this machine/instance. "
|
@@ -385,8 +588,8 @@ class UnsupportedSchedulerError(ResourceValidationError):
|
|
385
588
|
)
|
386
589
|
super().__init__(message)
|
387
590
|
self.scheduler = scheduler
|
388
|
-
self.supported = supported
|
389
|
-
self.available = available
|
591
|
+
self.supported = None if supported is None else tuple(supported)
|
592
|
+
self.available = None if available is None else tuple(available)
|
390
593
|
|
391
594
|
|
392
595
|
class UnknownSGEPEError(ResourceValidationError):
|
@@ -394,42 +597,84 @@ class UnknownSGEPEError(ResourceValidationError):
|
|
394
597
|
Miscellaneous error from SGE parallel environment.
|
395
598
|
"""
|
396
599
|
|
600
|
+
def __init__(self, env_name: str, all_env_names: Iterable[str]) -> None:
|
601
|
+
super().__init__(
|
602
|
+
f"The SGE parallel environment {env_name!r} is not "
|
603
|
+
f"specified in the configuration. Specified parallel environments "
|
604
|
+
f"are {sorted(all_env_names)!r}."
|
605
|
+
)
|
606
|
+
|
397
607
|
|
398
608
|
class IncompatibleSGEPEError(ResourceValidationError):
|
399
609
|
"""
|
400
610
|
The SGE parallel environment selected is incompatible.
|
401
611
|
"""
|
402
612
|
|
613
|
+
def __init__(self, env_name: str, num_cores: int | None) -> None:
|
614
|
+
super().__init__(
|
615
|
+
f"The SGE parallel environment {env_name!r} is not "
|
616
|
+
f"compatible with the number of cores requested: "
|
617
|
+
f"{num_cores!r}."
|
618
|
+
)
|
619
|
+
|
403
620
|
|
404
621
|
class NoCompatibleSGEPEError(ResourceValidationError):
|
405
622
|
"""
|
406
623
|
No SGE parallel environment is compatible with request.
|
407
624
|
"""
|
408
625
|
|
626
|
+
def __init__(self, num_cores: int | None) -> None:
|
627
|
+
super().__init__(
|
628
|
+
f"No compatible SGE parallel environment could be found for the "
|
629
|
+
f"specified `num_cores` ({num_cores!r})."
|
630
|
+
)
|
631
|
+
|
409
632
|
|
410
633
|
class IncompatibleParallelModeError(ResourceValidationError):
|
411
634
|
"""
|
412
635
|
The parallel mode is incompatible.
|
413
636
|
"""
|
414
637
|
|
638
|
+
def __init__(self, parallel_mode: ParallelMode) -> None:
|
639
|
+
super().__init__(
|
640
|
+
f"For the {parallel_mode.name.lower()} parallel mode, "
|
641
|
+
f"only a single node may be requested."
|
642
|
+
)
|
643
|
+
|
415
644
|
|
416
645
|
class UnknownSLURMPartitionError(ResourceValidationError):
|
417
646
|
"""
|
418
647
|
The requested SLURM partition isn't known.
|
419
648
|
"""
|
420
649
|
|
650
|
+
def __init__(self, part_name: str, all_parts: Iterable[str]) -> None:
|
651
|
+
super().__init__(
|
652
|
+
f"The SLURM partition {part_name!r} is not "
|
653
|
+
f"specified in the configuration. Specified partitions are "
|
654
|
+
f"{sorted(all_parts)!r}."
|
655
|
+
)
|
656
|
+
|
421
657
|
|
422
658
|
class IncompatibleSLURMPartitionError(ResourceValidationError):
|
423
659
|
"""
|
424
660
|
The requested SLURM partition is incompatible.
|
425
661
|
"""
|
426
662
|
|
663
|
+
def __init__(self, part_name: str, attr_kind: str, value) -> None:
|
664
|
+
super().__init__(
|
665
|
+
f"The SLURM partition {part_name!r} is not "
|
666
|
+
f"compatible with the {attr_kind} requested: {value!r}."
|
667
|
+
)
|
668
|
+
|
427
669
|
|
428
670
|
class IncompatibleSLURMArgumentsError(ResourceValidationError):
|
429
671
|
"""
|
430
672
|
The SLURM arguments are incompatible with each other.
|
431
673
|
"""
|
432
674
|
|
675
|
+
def __init__(self, msg: str) -> None:
|
676
|
+
super().__init__(msg)
|
677
|
+
|
433
678
|
|
434
679
|
class _MissingStoreItemError(ValueError):
|
435
680
|
def __init__(self, id_lst: Iterable[int], item_type: str) -> None:
|
@@ -490,18 +735,35 @@ class NotSubmitMachineError(RuntimeError):
|
|
490
735
|
The requested machine can't be submitted to.
|
491
736
|
"""
|
492
737
|
|
738
|
+
def __init__(self) -> None:
|
739
|
+
super().__init__(
|
740
|
+
"Cannot get active state of the jobscript because the current machine "
|
741
|
+
"is not the machine on which the jobscript was submitted."
|
742
|
+
)
|
743
|
+
|
493
744
|
|
494
745
|
class RunNotAbortableError(ValueError):
|
495
746
|
"""
|
496
747
|
Cannot abort the run.
|
497
748
|
"""
|
498
749
|
|
750
|
+
def __init__(self) -> None:
|
751
|
+
super().__init__(
|
752
|
+
"The run is not defined as abortable in the task schema, so it cannot "
|
753
|
+
"be aborted."
|
754
|
+
)
|
755
|
+
|
499
756
|
|
500
757
|
class NoCLIFormatMethodError(AttributeError):
|
501
758
|
"""
|
502
759
|
Some CLI class lacks a format method
|
503
760
|
"""
|
504
761
|
|
762
|
+
def __init__(self, method: str, inp_val: object) -> None:
|
763
|
+
super().__init__(
|
764
|
+
f"No CLI format method {method!r} exists for the object {inp_val!r}."
|
765
|
+
)
|
766
|
+
|
505
767
|
|
506
768
|
class ContainerKeyError(KeyError):
|
507
769
|
"""
|
@@ -509,11 +771,11 @@ class ContainerKeyError(KeyError):
|
|
509
771
|
|
510
772
|
Parameters
|
511
773
|
----------
|
512
|
-
path:
|
774
|
+
path:
|
513
775
|
The path whose resolution failed.
|
514
776
|
"""
|
515
777
|
|
516
|
-
def __init__(self, path:
|
778
|
+
def __init__(self, path: list[str]) -> None:
|
517
779
|
self.path = path
|
518
780
|
super().__init__()
|
519
781
|
|
@@ -524,11 +786,11 @@ class MayNeedObjectError(Exception):
|
|
524
786
|
|
525
787
|
Parameters
|
526
788
|
----------
|
527
|
-
path:
|
789
|
+
path:
|
528
790
|
The path whose resolution failed.
|
529
791
|
"""
|
530
792
|
|
531
|
-
def __init__(self, path):
|
793
|
+
def __init__(self, path: str) -> None:
|
532
794
|
self.path = path
|
533
795
|
super().__init__()
|
534
796
|
|
@@ -538,12 +800,18 @@ class NoAvailableElementSetsError(Exception):
|
|
538
800
|
No element set is available.
|
539
801
|
"""
|
540
802
|
|
803
|
+
def __init__(self) -> None:
|
804
|
+
super().__init__()
|
805
|
+
|
541
806
|
|
542
807
|
class OutputFileParserNoOutputError(ValueError):
|
543
808
|
"""
|
544
809
|
There was no output for the output file parser to parse.
|
545
810
|
"""
|
546
811
|
|
812
|
+
def __init__(self) -> None:
|
813
|
+
super().__init__()
|
814
|
+
|
547
815
|
|
548
816
|
class SubmissionEnvironmentError(ValueError):
|
549
817
|
"""
|
@@ -551,67 +819,146 @@ class SubmissionEnvironmentError(ValueError):
|
|
551
819
|
"""
|
552
820
|
|
553
821
|
|
822
|
+
def _spec_to_ref(env_spec: Mapping[str, Any]):
|
823
|
+
non_name_spec = {k: v for k, v in env_spec.items() if k != "name"}
|
824
|
+
spec_str = f" with specifiers {non_name_spec!r}" if non_name_spec else ""
|
825
|
+
return f"{env_spec['name']!r}{spec_str}"
|
826
|
+
|
827
|
+
|
554
828
|
class MissingEnvironmentExecutableError(SubmissionEnvironmentError):
|
555
829
|
"""
|
556
830
|
The environment does not have the requested executable at all.
|
557
831
|
"""
|
558
832
|
|
833
|
+
def __init__(self, env_spec: Mapping[str, Any], exec_label: str) -> None:
|
834
|
+
super().__init__(
|
835
|
+
f"The environment {_spec_to_ref(env_spec)} as defined on this machine has no "
|
836
|
+
f"executable labelled {exec_label!r}, which is required for this "
|
837
|
+
f"submission, so the submission cannot be created."
|
838
|
+
)
|
839
|
+
|
559
840
|
|
560
841
|
class MissingEnvironmentExecutableInstanceError(SubmissionEnvironmentError):
|
561
842
|
"""
|
562
843
|
The environment does not have a suitable instance of the requested executable.
|
563
844
|
"""
|
564
845
|
|
846
|
+
def __init__(
|
847
|
+
self, env_spec: Mapping[str, Any], exec_label: str, js_idx: int, res: dict
|
848
|
+
) -> None:
|
849
|
+
super().__init__(
|
850
|
+
f"No matching executable instances found for executable "
|
851
|
+
f"{exec_label!r} of environment {_spec_to_ref(env_spec)} for jobscript "
|
852
|
+
f"index {js_idx!r} with requested resources {res!r}."
|
853
|
+
)
|
854
|
+
|
565
855
|
|
566
856
|
class MissingEnvironmentError(SubmissionEnvironmentError):
|
567
857
|
"""
|
568
858
|
There is no environment with that name.
|
569
859
|
"""
|
570
860
|
|
861
|
+
def __init__(self, env_spec: Mapping[str, Any]) -> None:
|
862
|
+
super().__init__(
|
863
|
+
f"The environment {_spec_to_ref(env_spec)} is not defined on this machine, so the "
|
864
|
+
f"submission cannot be created."
|
865
|
+
)
|
866
|
+
|
571
867
|
|
572
868
|
class UnsupportedScriptDataFormat(ValueError):
|
573
869
|
"""
|
574
870
|
That format of script data is not supported.
|
575
871
|
"""
|
576
872
|
|
873
|
+
def __init__(
|
874
|
+
self, data: ScriptData, kind: str, name: str, formats: tuple[str, ...]
|
875
|
+
) -> None:
|
876
|
+
super().__init__(
|
877
|
+
f"Script data format {data!r} for {kind} parameter {name!r} is not "
|
878
|
+
f"understood. Available script data formats are: "
|
879
|
+
f"{formats!r}."
|
880
|
+
)
|
881
|
+
|
577
882
|
|
578
883
|
class UnknownScriptDataParameter(ValueError):
|
579
884
|
"""
|
580
885
|
Unknown parameter in script data.
|
581
886
|
"""
|
582
887
|
|
888
|
+
def __init__(self, name: str, kind: str, param_names: Sequence[str]) -> None:
|
889
|
+
super().__init__(
|
890
|
+
f"Script data parameter {name!r} is not a known parameter of the "
|
891
|
+
f"action. Parameters ({kind}) are: {param_names!r}."
|
892
|
+
)
|
893
|
+
|
583
894
|
|
584
895
|
class UnknownScriptDataKey(ValueError):
|
585
896
|
"""
|
586
897
|
Unknown key in script data.
|
587
898
|
"""
|
588
899
|
|
900
|
+
def __init__(self, key: str, allowed_keys: Sequence[str]) -> None:
|
901
|
+
super().__init__(
|
902
|
+
f"Script data key {key!r} is not understood. Allowed keys are: "
|
903
|
+
f"{allowed_keys!r}."
|
904
|
+
)
|
905
|
+
|
589
906
|
|
590
907
|
class MissingVariableSubstitutionError(KeyError):
|
591
908
|
"""
|
592
909
|
No definition available of a variable being substituted.
|
593
910
|
"""
|
594
911
|
|
912
|
+
def __init__(self, var_name: str, variables: Iterable[str]) -> None:
|
913
|
+
super().__init__(
|
914
|
+
f"The variable {var_name!r} referenced in the string does not match "
|
915
|
+
f"any of the provided variables: {sorted(variables)!r}."
|
916
|
+
)
|
917
|
+
|
595
918
|
|
596
919
|
class EnvironmentPresetUnknownEnvironmentError(ValueError):
|
597
920
|
"""
|
598
921
|
An environment preset could not be resolved to an execution environment.
|
599
922
|
"""
|
600
923
|
|
924
|
+
def __init__(self, name: str, bad_envs: Iterable[str]) -> None:
|
925
|
+
super().__init__(
|
926
|
+
f"Task schema {name} has environment presets that refer to one "
|
927
|
+
f"or more environments that are not referenced in any of the task "
|
928
|
+
f"schema's actions: {', '.join(f'{env!r}' for env in sorted(bad_envs))}."
|
929
|
+
)
|
930
|
+
|
601
931
|
|
602
932
|
class UnknownEnvironmentPresetError(ValueError):
|
603
933
|
"""
|
604
934
|
An execution environment was unknown.
|
605
935
|
"""
|
606
936
|
|
937
|
+
def __init__(self, preset_name: str, schema_name: str) -> None:
|
938
|
+
super().__init__(
|
939
|
+
f"There is no environment preset named {preset_name!r} defined "
|
940
|
+
f"in the task schema {schema_name}."
|
941
|
+
)
|
942
|
+
|
607
943
|
|
608
944
|
class MultipleEnvironmentsError(ValueError):
|
609
945
|
"""
|
610
946
|
Multiple applicable execution environments exist.
|
611
947
|
"""
|
612
948
|
|
949
|
+
def __init__(self, env_spec: Mapping[str, Any]) -> None:
|
950
|
+
super().__init__(
|
951
|
+
f"Multiple environments {_spec_to_ref(env_spec)} are defined on this machine."
|
952
|
+
)
|
953
|
+
|
613
954
|
|
614
955
|
class MissingElementGroup(ValueError):
|
615
956
|
"""
|
616
957
|
An element group should exist but doesn't.
|
617
958
|
"""
|
959
|
+
|
960
|
+
def __init__(self, task_name: str, group_name: str, input_path: str) -> None:
|
961
|
+
super().__init__(
|
962
|
+
f"Adding elements to task {task_name!r}: "
|
963
|
+
f"no element group named {group_name!r} found for input {input_path!r}."
|
964
|
+
)
|