hpcflow-new2 0.2.0a188__py3-none-any.whl → 0.2.0a190__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. hpcflow/__pyinstaller/hook-hpcflow.py +8 -6
  2. hpcflow/_version.py +1 -1
  3. hpcflow/app.py +1 -0
  4. hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
  5. hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
  6. hpcflow/sdk/__init__.py +21 -15
  7. hpcflow/sdk/app.py +2133 -770
  8. hpcflow/sdk/cli.py +281 -250
  9. hpcflow/sdk/cli_common.py +6 -2
  10. hpcflow/sdk/config/__init__.py +1 -1
  11. hpcflow/sdk/config/callbacks.py +77 -42
  12. hpcflow/sdk/config/cli.py +126 -103
  13. hpcflow/sdk/config/config.py +578 -311
  14. hpcflow/sdk/config/config_file.py +131 -95
  15. hpcflow/sdk/config/errors.py +112 -85
  16. hpcflow/sdk/config/types.py +145 -0
  17. hpcflow/sdk/core/actions.py +1054 -994
  18. hpcflow/sdk/core/app_aware.py +24 -0
  19. hpcflow/sdk/core/cache.py +81 -63
  20. hpcflow/sdk/core/command_files.py +275 -185
  21. hpcflow/sdk/core/commands.py +111 -107
  22. hpcflow/sdk/core/element.py +724 -503
  23. hpcflow/sdk/core/enums.py +192 -0
  24. hpcflow/sdk/core/environment.py +74 -93
  25. hpcflow/sdk/core/errors.py +398 -51
  26. hpcflow/sdk/core/json_like.py +540 -272
  27. hpcflow/sdk/core/loop.py +380 -334
  28. hpcflow/sdk/core/loop_cache.py +160 -43
  29. hpcflow/sdk/core/object_list.py +370 -207
  30. hpcflow/sdk/core/parameters.py +728 -600
  31. hpcflow/sdk/core/rule.py +59 -41
  32. hpcflow/sdk/core/run_dir_files.py +33 -22
  33. hpcflow/sdk/core/task.py +1546 -1325
  34. hpcflow/sdk/core/task_schema.py +240 -196
  35. hpcflow/sdk/core/test_utils.py +126 -88
  36. hpcflow/sdk/core/types.py +387 -0
  37. hpcflow/sdk/core/utils.py +410 -305
  38. hpcflow/sdk/core/validation.py +82 -9
  39. hpcflow/sdk/core/workflow.py +1192 -1028
  40. hpcflow/sdk/core/zarr_io.py +98 -137
  41. hpcflow/sdk/demo/cli.py +46 -33
  42. hpcflow/sdk/helper/cli.py +18 -16
  43. hpcflow/sdk/helper/helper.py +75 -63
  44. hpcflow/sdk/helper/watcher.py +61 -28
  45. hpcflow/sdk/log.py +83 -59
  46. hpcflow/sdk/persistence/__init__.py +8 -31
  47. hpcflow/sdk/persistence/base.py +988 -586
  48. hpcflow/sdk/persistence/defaults.py +6 -0
  49. hpcflow/sdk/persistence/discovery.py +38 -0
  50. hpcflow/sdk/persistence/json.py +408 -153
  51. hpcflow/sdk/persistence/pending.py +158 -123
  52. hpcflow/sdk/persistence/store_resource.py +37 -22
  53. hpcflow/sdk/persistence/types.py +307 -0
  54. hpcflow/sdk/persistence/utils.py +14 -11
  55. hpcflow/sdk/persistence/zarr.py +477 -420
  56. hpcflow/sdk/runtime.py +44 -41
  57. hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
  58. hpcflow/sdk/submission/jobscript.py +444 -404
  59. hpcflow/sdk/submission/schedulers/__init__.py +133 -40
  60. hpcflow/sdk/submission/schedulers/direct.py +97 -71
  61. hpcflow/sdk/submission/schedulers/sge.py +132 -126
  62. hpcflow/sdk/submission/schedulers/slurm.py +263 -268
  63. hpcflow/sdk/submission/schedulers/utils.py +7 -2
  64. hpcflow/sdk/submission/shells/__init__.py +14 -15
  65. hpcflow/sdk/submission/shells/base.py +102 -29
  66. hpcflow/sdk/submission/shells/bash.py +72 -55
  67. hpcflow/sdk/submission/shells/os_version.py +31 -30
  68. hpcflow/sdk/submission/shells/powershell.py +37 -29
  69. hpcflow/sdk/submission/submission.py +203 -257
  70. hpcflow/sdk/submission/types.py +143 -0
  71. hpcflow/sdk/typing.py +163 -12
  72. hpcflow/tests/conftest.py +8 -6
  73. hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
  74. hpcflow/tests/scripts/test_main_scripts.py +60 -30
  75. hpcflow/tests/shells/wsl/test_wsl_submission.py +6 -4
  76. hpcflow/tests/unit/test_action.py +86 -75
  77. hpcflow/tests/unit/test_action_rule.py +9 -4
  78. hpcflow/tests/unit/test_app.py +13 -6
  79. hpcflow/tests/unit/test_cli.py +1 -1
  80. hpcflow/tests/unit/test_command.py +71 -54
  81. hpcflow/tests/unit/test_config.py +20 -15
  82. hpcflow/tests/unit/test_config_file.py +21 -18
  83. hpcflow/tests/unit/test_element.py +58 -62
  84. hpcflow/tests/unit/test_element_iteration.py +3 -1
  85. hpcflow/tests/unit/test_element_set.py +29 -19
  86. hpcflow/tests/unit/test_group.py +4 -2
  87. hpcflow/tests/unit/test_input_source.py +116 -93
  88. hpcflow/tests/unit/test_input_value.py +29 -24
  89. hpcflow/tests/unit/test_json_like.py +44 -35
  90. hpcflow/tests/unit/test_loop.py +65 -58
  91. hpcflow/tests/unit/test_object_list.py +17 -12
  92. hpcflow/tests/unit/test_parameter.py +16 -7
  93. hpcflow/tests/unit/test_persistence.py +48 -35
  94. hpcflow/tests/unit/test_resources.py +20 -18
  95. hpcflow/tests/unit/test_run.py +8 -3
  96. hpcflow/tests/unit/test_runtime.py +2 -1
  97. hpcflow/tests/unit/test_schema_input.py +23 -15
  98. hpcflow/tests/unit/test_shell.py +3 -2
  99. hpcflow/tests/unit/test_slurm.py +8 -7
  100. hpcflow/tests/unit/test_submission.py +39 -19
  101. hpcflow/tests/unit/test_task.py +352 -247
  102. hpcflow/tests/unit/test_task_schema.py +33 -20
  103. hpcflow/tests/unit/test_utils.py +9 -11
  104. hpcflow/tests/unit/test_value_sequence.py +15 -12
  105. hpcflow/tests/unit/test_workflow.py +114 -83
  106. hpcflow/tests/unit/test_workflow_template.py +0 -1
  107. hpcflow/tests/workflows/test_jobscript.py +2 -1
  108. hpcflow/tests/workflows/test_workflows.py +18 -13
  109. {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/METADATA +2 -1
  110. hpcflow_new2-0.2.0a190.dist-info/RECORD +165 -0
  111. hpcflow/sdk/core/parallel.py +0 -21
  112. hpcflow_new2-0.2.0a188.dist-info/RECORD +0 -158
  113. {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/LICENSE +0 -0
  114. {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/WHEEL +0 -0
  115. {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/entry_points.txt +0 -0
@@ -2,8 +2,17 @@
2
2
  Errors from the workflow system.
3
3
  """
4
4
 
5
+ from __future__ import annotations
5
6
  import os
6
- from typing import Iterable, List
7
+ from collections.abc import Iterable, Mapping, Sequence
8
+ from textwrap import indent
9
+ from typing import Any, TYPE_CHECKING
10
+
11
+ if TYPE_CHECKING:
12
+ from .enums import ParallelMode
13
+ from .object_list import WorkflowLoopList
14
+ from .parameters import InputSource, ValueSequence
15
+ from .types import ScriptData
7
16
 
8
17
 
9
18
  class InputValueDuplicateSequenceAddress(ValueError):
@@ -11,36 +20,65 @@ class InputValueDuplicateSequenceAddress(ValueError):
11
20
  An InputValue has the same sequence address twice.
12
21
  """
13
22
 
23
+ # FIXME: never used
24
+
14
25
 
15
26
  class TaskTemplateMultipleSchemaObjectives(ValueError):
16
27
  """
17
28
  A TaskTemplate has multiple objectives.
18
29
  """
19
30
 
31
+ def __init__(self, names: set[str]) -> None:
32
+ super().__init__(
33
+ f"All task schemas used within a task must have the same "
34
+ f"objective, but found multiple objectives: {sorted(names)!r}"
35
+ )
36
+
20
37
 
21
38
  class TaskTemplateUnexpectedInput(ValueError):
22
39
  """
23
40
  A TaskTemplate was given unexpected input.
24
41
  """
25
42
 
43
+ def __init__(self, unexpected_types: set[str]) -> None:
44
+ super().__init__(
45
+ f"The following input parameters are unexpected: {sorted(unexpected_types)!r}"
46
+ )
47
+
26
48
 
27
49
  class TaskTemplateUnexpectedSequenceInput(ValueError):
28
50
  """
29
51
  A TaskTemplate was given an unexpected sequence.
30
52
  """
31
53
 
54
+ def __init__(
55
+ self, inp_type: str, expected_types: set[str], seq: ValueSequence
56
+ ) -> None:
57
+ allowed_str = ", ".join(f'"{in_typ}"' for in_typ in expected_types)
58
+ super().__init__(
59
+ f"The input type {inp_type!r} specified in the following sequence"
60
+ f" path is unexpected: {seq.path!r}. Available input types are: "
61
+ f"{allowed_str}."
62
+ )
63
+
32
64
 
33
65
  class TaskTemplateMultipleInputValues(ValueError):
34
66
  """
35
67
  A TaskTemplate had multiple input values bound over each other.
36
68
  """
37
69
 
70
+ def __init__(self, msg: str) -> None:
71
+ super().__init__(msg)
72
+
38
73
 
39
74
  class InvalidIdentifier(ValueError):
40
75
  """
41
76
  A bad identifier name was given.
42
77
  """
43
78
 
79
+ def __init__(self, name: str) -> None:
80
+ super().__init__(f"Invalid string for identifier: {name!r}")
81
+
44
82
 
45
83
  class MissingInputs(Exception):
46
84
  """
@@ -48,17 +86,16 @@ class MissingInputs(Exception):
48
86
 
49
87
  Parameters
50
88
  ----------
51
- message: str
52
- The message of the exception.
53
- missing_inputs: list[str]
89
+ missing_inputs:
54
90
  The missing inputs.
55
91
  """
56
92
 
57
93
  # TODO: add links to doc pages for common user-exceptions?
58
94
 
59
- def __init__(self, message, missing_inputs) -> None:
60
- self.missing_inputs = missing_inputs
61
- super().__init__(message)
95
+ def __init__(self, missing_inputs: Iterable[str]) -> None:
96
+ self.missing_inputs = tuple(missing_inputs)
97
+ missing_str = ", ".join(map(repr, missing_inputs))
98
+ super().__init__(f"The following inputs have no sources: {missing_str}.")
62
99
 
63
100
 
64
101
  class UnrequiredInputSources(ValueError):
@@ -67,23 +104,23 @@ class UnrequiredInputSources(ValueError):
67
104
 
68
105
  Parameters
69
106
  ----------
70
- message: str
71
- The message of the exception.
72
- unrequired_sources: list
107
+ unrequired_sources:
73
108
  The input sources that were not required.
74
109
  """
75
110
 
76
- def __init__(self, message, unrequired_sources) -> None:
77
- self.unrequired_sources = unrequired_sources
78
- for src in unrequired_sources:
79
- if src.startswith("inputs."):
80
- # reminder about how to specify input sources:
81
- message += (
82
- f" Note that input source keys should not be specified with the "
83
- f"'inputs.' prefix. Did you mean to specify {src[len('inputs.'):]!r} "
84
- f"instead of {src!r}?"
85
- )
86
- break
111
+ def __init__(self, unrequired_sources: Iterable[str]) -> None:
112
+ self.unrequired_sources = frozenset(unrequired_sources)
113
+ message = (
114
+ f"The following input sources are not required but have been specified: "
115
+ f'{", ".join(map(repr, sorted(self.unrequired_sources)))}.'
116
+ )
117
+ if any((bad := src).startswith("inputs.") for src in self.unrequired_sources):
118
+ # reminder about how to specify input sources:
119
+ message += (
120
+ f" Note that input source keys should not be specified with the "
121
+ f"'inputs.' prefix. Did you mean to specify "
122
+ f"{bad.removeprefix('inputs.')!r} instead of {bad!r}?"
123
+ )
87
124
  super().__init__(message)
88
125
 
89
126
 
@@ -93,15 +130,16 @@ class ExtraInputs(Exception):
93
130
 
94
131
  Parameters
95
132
  ----------
96
- message: str
97
- The message of the exception.
98
- extra_inputs: list
133
+ extra_inputs:
99
134
  The extra inputs.
100
135
  """
101
136
 
102
- def __init__(self, message, extra_inputs) -> None:
103
- self.extra_inputs = extra_inputs
104
- super().__init__(message)
137
+ def __init__(self, extra_inputs: set[str]) -> None:
138
+ self.extra_inputs = frozenset(extra_inputs)
139
+ super().__init__(
140
+ f"The following inputs are not required, but have been passed: "
141
+ f'{", ".join(f"{typ!r}" for typ in extra_inputs)}.'
142
+ )
105
143
 
106
144
 
107
145
  class UnavailableInputSource(ValueError):
@@ -109,184 +147,311 @@ class UnavailableInputSource(ValueError):
109
147
  An input source was not available.
110
148
  """
111
149
 
150
+ def __init__(
151
+ self, source: InputSource, path: str, avail: Sequence[InputSource]
152
+ ) -> None:
153
+ super().__init__(
154
+ f"The input source {source.to_string()!r} is not "
155
+ f"available for input path {path!r}. Available "
156
+ f"input sources are: {[src.to_string() for src in avail]}."
157
+ )
158
+
112
159
 
113
160
  class InapplicableInputSourceElementIters(ValueError):
114
161
  """
115
162
  An input source element iteration was inapplicable."""
116
163
 
164
+ def __init__(self, source: InputSource, elem_iters_IDs: Sequence[int] | None) -> None:
165
+ super().__init__(
166
+ f"The specified `element_iters` for input source "
167
+ f"{source.to_string()!r} are not all applicable. "
168
+ f"Applicable element iteration IDs for this input source "
169
+ f"are: {elem_iters_IDs!r}."
170
+ )
171
+
117
172
 
118
173
  class NoCoincidentInputSources(ValueError):
119
174
  """
120
175
  Could not line up input sources to make an actual valid execution.
121
176
  """
122
177
 
178
+ def __init__(self, name: str, task_ref: int) -> None:
179
+ super().__init__(
180
+ f"Task {name!r}: input sources from task {task_ref!r} have "
181
+ f"no coincident applicable element iterations. Consider setting "
182
+ f"the element set (or task) argument "
183
+ f"`allow_non_coincident_task_sources` to `True`, which will "
184
+ f"allow for input sources from the same task to use different "
185
+ f"(non-coinciding) subsets of element iterations from the "
186
+ f"source task."
187
+ )
188
+
123
189
 
124
190
  class TaskTemplateInvalidNesting(ValueError):
125
191
  """
126
192
  Invalid nesting in a task template.
127
193
  """
128
194
 
195
+ def __init__(self, key: str, value: float) -> None:
196
+ super().__init__(
197
+ f"`nesting_order` must be >=0 for all keys, but for key {key!r}, value "
198
+ f"of {value!r} was specified."
199
+ )
200
+
129
201
 
130
202
  class TaskSchemaSpecValidationError(Exception):
131
203
  """
132
204
  A task schema failed to validate.
133
205
  """
134
206
 
207
+ # FIXME: never used
208
+
135
209
 
136
210
  class WorkflowSpecValidationError(Exception):
137
211
  """
138
212
  A workflow failed to validate.
139
213
  """
140
214
 
215
+ # FIXME: never used
216
+
141
217
 
142
218
  class InputSourceValidationError(Exception):
143
219
  """
144
220
  An input source failed to validate.
145
221
  """
146
222
 
223
+ # FIXME: never used
224
+
147
225
 
148
226
  class EnvironmentSpecValidationError(Exception):
149
227
  """
150
228
  An environment specification failed to validate.
151
229
  """
152
230
 
231
+ # FIXME: never used
232
+
153
233
 
154
234
  class ParameterSpecValidationError(Exception):
155
235
  """
156
236
  A parameter specification failed to validate.
157
237
  """
158
238
 
239
+ # FIXME: never used
240
+
159
241
 
160
242
  class FileSpecValidationError(Exception):
161
243
  """
162
244
  A file specification failed to validate.
163
245
  """
164
246
 
247
+ # FIXME: never used
248
+
165
249
 
166
250
  class DuplicateExecutableError(ValueError):
167
251
  """
168
252
  The same executable was present twice in an executable environment.
169
253
  """
170
254
 
255
+ def __init__(self, duplicate_labels: list) -> None:
256
+ super().__init__(
257
+ f"Executables must have unique `label`s within each environment, but "
258
+ f"found label(s) multiple times: {duplicate_labels!r}"
259
+ )
260
+
171
261
 
172
262
  class MissingCompatibleActionEnvironment(Exception):
173
263
  """
174
264
  Could not find a compatible action environment.
175
265
  """
176
266
 
267
+ def __init__(self, msg: str) -> None:
268
+ super().__init__(f"No compatible environment is specified for the {msg}.")
269
+
177
270
 
178
271
  class MissingActionEnvironment(Exception):
179
272
  """
180
273
  Could not find an action environment.
181
274
  """
182
275
 
276
+ # FIXME: never used
277
+
183
278
 
184
279
  class ActionEnvironmentMissingNameError(Exception):
185
280
  """
186
281
  An action environment was missing its name.
187
282
  """
188
283
 
284
+ def __init__(self, environment: Mapping[str, Any]) -> None:
285
+ super().__init__(
286
+ "The action-environment environment specification must include a string "
287
+ "`name` key, or be specified as string that is that name. Provided "
288
+ f"environment key was {environment!r}."
289
+ )
290
+
189
291
 
190
292
  class FromSpecMissingObjectError(Exception):
191
293
  """
192
294
  Missing object when deserialising from specification.
193
295
  """
194
296
 
297
+ # FIXME: never used
298
+
195
299
 
196
300
  class TaskSchemaMissingParameterError(Exception):
197
301
  """
198
302
  Parameter was missing from task schema.
199
303
  """
200
304
 
305
+ # FIXME: never used
306
+
201
307
 
202
308
  class ToJSONLikeChildReferenceError(Exception):
203
309
  """
204
310
  Failed to generate or reference a child object when converting to JSON.
205
311
  """
206
312
 
313
+ # FIXME: never thrown
314
+
207
315
 
208
316
  class InvalidInputSourceTaskReference(Exception):
209
317
  """
210
318
  Invalid input source in task reference.
211
319
  """
212
320
 
321
+ def __init__(self, input_source: InputSource, task_ref: int | None = None) -> None:
322
+ super().__init__(
323
+ f"Input source {input_source.to_string()!r} cannot refer to the "
324
+ f"outputs of its own task!"
325
+ if task_ref is None
326
+ else f"Input source {input_source.to_string()!r} refers to a missing "
327
+ f"or inaccessible task: {task_ref!r}."
328
+ )
329
+
213
330
 
214
331
  class WorkflowNotFoundError(Exception):
215
332
  """
216
333
  Could not find the workflow.
217
334
  """
218
335
 
336
+ def __init__(self, path, fs) -> None:
337
+ super().__init__(
338
+ f"Cannot infer a store format at path {path!r} with file system {fs!r}."
339
+ )
340
+
219
341
 
220
342
  class MalformedWorkflowError(Exception):
221
343
  """
222
344
  Workflow was a malformed document.
223
345
  """
224
346
 
347
+ # TODO: use this class
348
+
225
349
 
226
350
  class ValuesAlreadyPersistentError(Exception):
227
351
  """
228
352
  Trying to make a value persistent that already is so.
229
353
  """
230
354
 
355
+ # FIXME: never used
356
+
231
357
 
232
358
  class MalformedParameterPathError(ValueError):
233
359
  """
234
360
  The path to a parameter was ill-formed.
235
361
  """
236
362
 
363
+ def __init__(self, msg: str) -> None:
364
+ super().__init__(msg)
365
+
237
366
 
238
367
  class MalformedNestingOrderPath(ValueError):
239
368
  """
240
369
  A nesting order path was ill-formed.
241
370
  """
242
371
 
372
+ def __init__(self, k: str, allowed_nesting_paths: Iterable[str]) -> None:
373
+ super().__init__(
374
+ f"Element set: nesting order path {k!r} not understood. Each key in "
375
+ f"`nesting_order` must be start with one of "
376
+ f"{sorted(allowed_nesting_paths)!r}."
377
+ )
378
+
243
379
 
244
380
  class UnknownResourceSpecItemError(ValueError):
245
381
  """
246
382
  A resource specification item was not found.
247
383
  """
248
384
 
385
+ def __init__(self, msg: str) -> None:
386
+ super().__init__(msg)
387
+
249
388
 
250
389
  class WorkflowParameterMissingError(AttributeError):
251
390
  """
252
391
  A parameter to a workflow was missing.
253
392
  """
254
393
 
394
+ # FIXME: never thrown
395
+
255
396
 
256
397
  class WorkflowBatchUpdateFailedError(Exception):
257
398
  """
258
399
  An update to a workflow failed.
259
400
  """
260
401
 
402
+ # FIXME: only throw is commented out?
403
+
261
404
 
262
405
  class WorkflowLimitsError(ValueError):
263
406
  """
264
407
  Workflow hit limits.
265
408
  """
266
409
 
410
+ # FIXME: never used
411
+
267
412
 
268
413
  class UnsetParameterDataError(Exception):
269
414
  """
270
415
  Tried to read from an unset parameter.
271
416
  """
272
417
 
418
+ def __init__(self, path: str | None, path_i: str) -> None:
419
+ super().__init__(
420
+ f"Element data path {path!r} resolves to unset data for "
421
+ f"(at least) data-index path: {path_i!r}."
422
+ )
423
+
273
424
 
274
425
  class LoopAlreadyExistsError(Exception):
275
426
  """
276
427
  A particular loop (or its name) already exists.
277
428
  """
278
429
 
430
+ def __init__(self, loop_name: str, loops: WorkflowLoopList) -> None:
431
+ super().__init__(
432
+ f"A loop with the name {loop_name!r} already exists in the workflow: "
433
+ f"{getattr(loops, loop_name)!r}."
434
+ )
435
+
279
436
 
280
437
  class LoopTaskSubsetError(ValueError):
281
438
  """
282
439
  Problem constructing a subset of a task for a loop.
283
440
  """
284
441
 
442
+ def __init__(self, loop_name: str, task_indices: Sequence[int]) -> None:
443
+ super().__init__(
444
+ f"Loop {loop_name!r}: task subset must be an ascending contiguous range, "
445
+ f"but specified task indices were: {task_indices!r}."
446
+ )
447
+
285
448
 
286
449
  class SchedulerVersionsFailure(RuntimeError):
287
450
  """We couldn't get the scheduler and or shell versions."""
288
451
 
289
- def __init__(self, message):
452
+ # FIXME: unused
453
+
454
+ def __init__(self, message: str) -> None:
290
455
  self.message = message
291
456
  super().__init__(message)
292
457
 
@@ -298,15 +463,16 @@ class JobscriptSubmissionFailure(RuntimeError):
298
463
 
299
464
  def __init__(
300
465
  self,
301
- message,
302
- submit_cmd,
303
- js_idx,
304
- js_path,
305
- stdout,
306
- stderr,
307
- subprocess_exc,
308
- job_ID_parse_exc,
309
- ) -> None:
466
+ message: str,
467
+ *,
468
+ submit_cmd: list[str],
469
+ js_idx: int,
470
+ js_path: str,
471
+ stdout: str | None = None,
472
+ stderr: str | None = None,
473
+ subprocess_exc: Exception | None = None,
474
+ job_ID_parse_exc: Exception | None = None,
475
+ ):
310
476
  self.message = message
311
477
  self.submit_cmd = submit_cmd
312
478
  self.js_idx = js_idx
@@ -323,9 +489,37 @@ class SubmissionFailure(RuntimeError):
323
489
  A job submission failed.
324
490
  """
325
491
 
326
- def __init__(self, message) -> None:
327
- self.message = message
328
- super().__init__(message)
492
+ def __init__(
493
+ self,
494
+ sub_idx: int,
495
+ submitted_js_idx: Sequence[int],
496
+ exceptions: Iterable[JobscriptSubmissionFailure],
497
+ ) -> None:
498
+ msg = f"Some jobscripts in submission index {sub_idx} could not be submitted"
499
+ if submitted_js_idx:
500
+ msg += f" (but jobscripts {submitted_js_idx} were submitted successfully):"
501
+ else:
502
+ msg += ":"
503
+
504
+ msg += "\n"
505
+ for sub_err in exceptions:
506
+ msg += (
507
+ f"Jobscript {sub_err.js_idx} at path: {str(sub_err.js_path)!r}\n"
508
+ f"Submit command: {sub_err.submit_cmd!r}.\n"
509
+ f"Reason: {sub_err.message!r}\n"
510
+ )
511
+ if sub_err.subprocess_exc is not None:
512
+ msg += f"Subprocess exception: {sub_err.subprocess_exc}\n"
513
+ if sub_err.job_ID_parse_exc is not None:
514
+ msg += f"Subprocess job ID parse exception: {sub_err.job_ID_parse_exc}\n"
515
+ if sub_err.job_ID_parse_exc is not None:
516
+ msg += f"Job ID parse exception: {sub_err.job_ID_parse_exc}\n"
517
+ if sub_err.stdout:
518
+ msg += f"Submission stdout:\n{indent(sub_err.stdout, ' ')}\n"
519
+ if sub_err.stderr:
520
+ msg += f"Submission stderr:\n{indent(sub_err.stderr, ' ')}\n"
521
+ self.message = msg
522
+ super().__init__(msg)
329
523
 
330
524
 
331
525
  class WorkflowSubmissionFailure(RuntimeError):
@@ -333,6 +527,9 @@ class WorkflowSubmissionFailure(RuntimeError):
333
527
  A workflow submission failed.
334
528
  """
335
529
 
530
+ def __init__(self, exceptions: Sequence[SubmissionFailure]) -> None:
531
+ super().__init__("\n" + "\n\n".join(exn.message for exn in exceptions))
532
+
336
533
 
337
534
  class ResourceValidationError(ValueError):
338
535
  """An incompatible resource requested by the user."""
@@ -341,7 +538,7 @@ class ResourceValidationError(ValueError):
341
538
  class UnsupportedOSError(ResourceValidationError):
342
539
  """This machine is not of the requested OS."""
343
540
 
344
- def __init__(self, os_name) -> None:
541
+ def __init__(self, os_name: str) -> None:
345
542
  message = (
346
543
  f"OS {os_name!r} is not compatible with this machine/instance with OS: "
347
544
  f"{os.name!r}."
@@ -353,14 +550,15 @@ class UnsupportedOSError(ResourceValidationError):
353
550
  class UnsupportedShellError(ResourceValidationError):
354
551
  """We don't support this shell on this OS."""
355
552
 
356
- def __init__(self, shell, supported) -> None:
553
+ def __init__(self, shell: str, supported: Iterable[str]) -> None:
554
+ sup = set(supported)
357
555
  message = (
358
556
  f"Shell {shell!r} is not supported on this machine/instance. Supported "
359
- f"shells are: {supported!r}."
557
+ f"shells are: {sup!r}."
360
558
  )
361
559
  super().__init__(message)
362
560
  self.shell = shell
363
- self.supported = supported
561
+ self.supported = frozenset(sup)
364
562
 
365
563
 
366
564
  class UnsupportedSchedulerError(ResourceValidationError):
@@ -371,7 +569,12 @@ class UnsupportedSchedulerError(ResourceValidationError):
371
569
 
372
570
  """
373
571
 
374
- def __init__(self, scheduler, supported=None, available=None) -> None:
572
+ def __init__(
573
+ self,
574
+ scheduler: str,
575
+ supported: Iterable[str] | None = None,
576
+ available: Iterable[str] | None = None,
577
+ ) -> None:
375
578
  if supported is not None:
376
579
  message = (
377
580
  f"Scheduler {scheduler!r} is not supported on this machine/instance. "
@@ -385,8 +588,8 @@ class UnsupportedSchedulerError(ResourceValidationError):
385
588
  )
386
589
  super().__init__(message)
387
590
  self.scheduler = scheduler
388
- self.supported = supported
389
- self.available = available
591
+ self.supported = None if supported is None else tuple(supported)
592
+ self.available = None if available is None else tuple(available)
390
593
 
391
594
 
392
595
  class UnknownSGEPEError(ResourceValidationError):
@@ -394,42 +597,84 @@ class UnknownSGEPEError(ResourceValidationError):
394
597
  Miscellaneous error from SGE parallel environment.
395
598
  """
396
599
 
600
+ def __init__(self, env_name: str, all_env_names: Iterable[str]) -> None:
601
+ super().__init__(
602
+ f"The SGE parallel environment {env_name!r} is not "
603
+ f"specified in the configuration. Specified parallel environments "
604
+ f"are {sorted(all_env_names)!r}."
605
+ )
606
+
397
607
 
398
608
  class IncompatibleSGEPEError(ResourceValidationError):
399
609
  """
400
610
  The SGE parallel environment selected is incompatible.
401
611
  """
402
612
 
613
+ def __init__(self, env_name: str, num_cores: int | None) -> None:
614
+ super().__init__(
615
+ f"The SGE parallel environment {env_name!r} is not "
616
+ f"compatible with the number of cores requested: "
617
+ f"{num_cores!r}."
618
+ )
619
+
403
620
 
404
621
  class NoCompatibleSGEPEError(ResourceValidationError):
405
622
  """
406
623
  No SGE parallel environment is compatible with request.
407
624
  """
408
625
 
626
+ def __init__(self, num_cores: int | None) -> None:
627
+ super().__init__(
628
+ f"No compatible SGE parallel environment could be found for the "
629
+ f"specified `num_cores` ({num_cores!r})."
630
+ )
631
+
409
632
 
410
633
  class IncompatibleParallelModeError(ResourceValidationError):
411
634
  """
412
635
  The parallel mode is incompatible.
413
636
  """
414
637
 
638
+ def __init__(self, parallel_mode: ParallelMode) -> None:
639
+ super().__init__(
640
+ f"For the {parallel_mode.name.lower()} parallel mode, "
641
+ f"only a single node may be requested."
642
+ )
643
+
415
644
 
416
645
  class UnknownSLURMPartitionError(ResourceValidationError):
417
646
  """
418
647
  The requested SLURM partition isn't known.
419
648
  """
420
649
 
650
+ def __init__(self, part_name: str, all_parts: Iterable[str]) -> None:
651
+ super().__init__(
652
+ f"The SLURM partition {part_name!r} is not "
653
+ f"specified in the configuration. Specified partitions are "
654
+ f"{sorted(all_parts)!r}."
655
+ )
656
+
421
657
 
422
658
  class IncompatibleSLURMPartitionError(ResourceValidationError):
423
659
  """
424
660
  The requested SLURM partition is incompatible.
425
661
  """
426
662
 
663
+ def __init__(self, part_name: str, attr_kind: str, value) -> None:
664
+ super().__init__(
665
+ f"The SLURM partition {part_name!r} is not "
666
+ f"compatible with the {attr_kind} requested: {value!r}."
667
+ )
668
+
427
669
 
428
670
  class IncompatibleSLURMArgumentsError(ResourceValidationError):
429
671
  """
430
672
  The SLURM arguments are incompatible with each other.
431
673
  """
432
674
 
675
+ def __init__(self, msg: str) -> None:
676
+ super().__init__(msg)
677
+
433
678
 
434
679
  class _MissingStoreItemError(ValueError):
435
680
  def __init__(self, id_lst: Iterable[int], item_type: str) -> None:
@@ -490,18 +735,35 @@ class NotSubmitMachineError(RuntimeError):
490
735
  The requested machine can't be submitted to.
491
736
  """
492
737
 
738
+ def __init__(self) -> None:
739
+ super().__init__(
740
+ "Cannot get active state of the jobscript because the current machine "
741
+ "is not the machine on which the jobscript was submitted."
742
+ )
743
+
493
744
 
494
745
  class RunNotAbortableError(ValueError):
495
746
  """
496
747
  Cannot abort the run.
497
748
  """
498
749
 
750
+ def __init__(self) -> None:
751
+ super().__init__(
752
+ "The run is not defined as abortable in the task schema, so it cannot "
753
+ "be aborted."
754
+ )
755
+
499
756
 
500
757
  class NoCLIFormatMethodError(AttributeError):
501
758
  """
502
759
  Some CLI class lacks a format method
503
760
  """
504
761
 
762
+ def __init__(self, method: str, inp_val: object) -> None:
763
+ super().__init__(
764
+ f"No CLI format method {method!r} exists for the object {inp_val!r}."
765
+ )
766
+
505
767
 
506
768
  class ContainerKeyError(KeyError):
507
769
  """
@@ -509,11 +771,11 @@ class ContainerKeyError(KeyError):
509
771
 
510
772
  Parameters
511
773
  ----------
512
- path: list[str]
774
+ path:
513
775
  The path whose resolution failed.
514
776
  """
515
777
 
516
- def __init__(self, path: List[str]) -> None:
778
+ def __init__(self, path: list[str]) -> None:
517
779
  self.path = path
518
780
  super().__init__()
519
781
 
@@ -524,11 +786,11 @@ class MayNeedObjectError(Exception):
524
786
 
525
787
  Parameters
526
788
  ----------
527
- path: list[str]
789
+ path:
528
790
  The path whose resolution failed.
529
791
  """
530
792
 
531
- def __init__(self, path):
793
+ def __init__(self, path: str) -> None:
532
794
  self.path = path
533
795
  super().__init__()
534
796
 
@@ -538,12 +800,18 @@ class NoAvailableElementSetsError(Exception):
538
800
  No element set is available.
539
801
  """
540
802
 
803
+ def __init__(self) -> None:
804
+ super().__init__()
805
+
541
806
 
542
807
  class OutputFileParserNoOutputError(ValueError):
543
808
  """
544
809
  There was no output for the output file parser to parse.
545
810
  """
546
811
 
812
+ def __init__(self) -> None:
813
+ super().__init__()
814
+
547
815
 
548
816
  class SubmissionEnvironmentError(ValueError):
549
817
  """
@@ -551,67 +819,146 @@ class SubmissionEnvironmentError(ValueError):
551
819
  """
552
820
 
553
821
 
822
+ def _spec_to_ref(env_spec: Mapping[str, Any]):
823
+ non_name_spec = {k: v for k, v in env_spec.items() if k != "name"}
824
+ spec_str = f" with specifiers {non_name_spec!r}" if non_name_spec else ""
825
+ return f"{env_spec['name']!r}{spec_str}"
826
+
827
+
554
828
  class MissingEnvironmentExecutableError(SubmissionEnvironmentError):
555
829
  """
556
830
  The environment does not have the requested executable at all.
557
831
  """
558
832
 
833
+ def __init__(self, env_spec: Mapping[str, Any], exec_label: str) -> None:
834
+ super().__init__(
835
+ f"The environment {_spec_to_ref(env_spec)} as defined on this machine has no "
836
+ f"executable labelled {exec_label!r}, which is required for this "
837
+ f"submission, so the submission cannot be created."
838
+ )
839
+
559
840
 
560
841
  class MissingEnvironmentExecutableInstanceError(SubmissionEnvironmentError):
561
842
  """
562
843
  The environment does not have a suitable instance of the requested executable.
563
844
  """
564
845
 
846
+ def __init__(
847
+ self, env_spec: Mapping[str, Any], exec_label: str, js_idx: int, res: dict
848
+ ) -> None:
849
+ super().__init__(
850
+ f"No matching executable instances found for executable "
851
+ f"{exec_label!r} of environment {_spec_to_ref(env_spec)} for jobscript "
852
+ f"index {js_idx!r} with requested resources {res!r}."
853
+ )
854
+
565
855
 
566
856
  class MissingEnvironmentError(SubmissionEnvironmentError):
567
857
  """
568
858
  There is no environment with that name.
569
859
  """
570
860
 
861
+ def __init__(self, env_spec: Mapping[str, Any]) -> None:
862
+ super().__init__(
863
+ f"The environment {_spec_to_ref(env_spec)} is not defined on this machine, so the "
864
+ f"submission cannot be created."
865
+ )
866
+
571
867
 
572
868
  class UnsupportedScriptDataFormat(ValueError):
573
869
  """
574
870
  That format of script data is not supported.
575
871
  """
576
872
 
873
+ def __init__(
874
+ self, data: ScriptData, kind: str, name: str, formats: tuple[str, ...]
875
+ ) -> None:
876
+ super().__init__(
877
+ f"Script data format {data!r} for {kind} parameter {name!r} is not "
878
+ f"understood. Available script data formats are: "
879
+ f"{formats!r}."
880
+ )
881
+
577
882
 
578
883
  class UnknownScriptDataParameter(ValueError):
579
884
  """
580
885
  Unknown parameter in script data.
581
886
  """
582
887
 
888
+ def __init__(self, name: str, kind: str, param_names: Sequence[str]) -> None:
889
+ super().__init__(
890
+ f"Script data parameter {name!r} is not a known parameter of the "
891
+ f"action. Parameters ({kind}) are: {param_names!r}."
892
+ )
893
+
583
894
 
584
895
  class UnknownScriptDataKey(ValueError):
585
896
  """
586
897
  Unknown key in script data.
587
898
  """
588
899
 
900
+ def __init__(self, key: str, allowed_keys: Sequence[str]) -> None:
901
+ super().__init__(
902
+ f"Script data key {key!r} is not understood. Allowed keys are: "
903
+ f"{allowed_keys!r}."
904
+ )
905
+
589
906
 
590
907
  class MissingVariableSubstitutionError(KeyError):
591
908
  """
592
909
  No definition available of a variable being substituted.
593
910
  """
594
911
 
912
+ def __init__(self, var_name: str, variables: Iterable[str]) -> None:
913
+ super().__init__(
914
+ f"The variable {var_name!r} referenced in the string does not match "
915
+ f"any of the provided variables: {sorted(variables)!r}."
916
+ )
917
+
595
918
 
596
919
  class EnvironmentPresetUnknownEnvironmentError(ValueError):
597
920
  """
598
921
  An environment preset could not be resolved to an execution environment.
599
922
  """
600
923
 
924
+ def __init__(self, name: str, bad_envs: Iterable[str]) -> None:
925
+ super().__init__(
926
+ f"Task schema {name} has environment presets that refer to one "
927
+ f"or more environments that are not referenced in any of the task "
928
+ f"schema's actions: {', '.join(f'{env!r}' for env in sorted(bad_envs))}."
929
+ )
930
+
601
931
 
602
932
  class UnknownEnvironmentPresetError(ValueError):
603
933
  """
604
934
  An execution environment was unknown.
605
935
  """
606
936
 
937
+ def __init__(self, preset_name: str, schema_name: str) -> None:
938
+ super().__init__(
939
+ f"There is no environment preset named {preset_name!r} defined "
940
+ f"in the task schema {schema_name}."
941
+ )
942
+
607
943
 
608
944
  class MultipleEnvironmentsError(ValueError):
609
945
  """
610
946
  Multiple applicable execution environments exist.
611
947
  """
612
948
 
949
+ def __init__(self, env_spec: Mapping[str, Any]) -> None:
950
+ super().__init__(
951
+ f"Multiple environments {_spec_to_ref(env_spec)} are defined on this machine."
952
+ )
953
+
613
954
 
614
955
  class MissingElementGroup(ValueError):
615
956
  """
616
957
  An element group should exist but doesn't.
617
958
  """
959
+
960
+ def __init__(self, task_name: str, group_name: str, input_path: str) -> None:
961
+ super().__init__(
962
+ f"Adding elements to task {task_name!r}: "
963
+ f"no element group named {group_name!r} found for input {input_path!r}."
964
+ )