hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a180__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. hpcflow/_version.py +1 -1
  2. hpcflow/data/demo_data_manifest/__init__.py +3 -0
  3. hpcflow/sdk/__init__.py +4 -1
  4. hpcflow/sdk/app.py +160 -15
  5. hpcflow/sdk/cli.py +14 -0
  6. hpcflow/sdk/cli_common.py +83 -0
  7. hpcflow/sdk/config/__init__.py +4 -0
  8. hpcflow/sdk/config/callbacks.py +25 -2
  9. hpcflow/sdk/config/cli.py +4 -1
  10. hpcflow/sdk/config/config.py +188 -14
  11. hpcflow/sdk/config/config_file.py +91 -3
  12. hpcflow/sdk/config/errors.py +33 -0
  13. hpcflow/sdk/core/__init__.py +2 -0
  14. hpcflow/sdk/core/actions.py +492 -35
  15. hpcflow/sdk/core/cache.py +22 -0
  16. hpcflow/sdk/core/command_files.py +221 -5
  17. hpcflow/sdk/core/commands.py +57 -0
  18. hpcflow/sdk/core/element.py +407 -8
  19. hpcflow/sdk/core/environment.py +92 -0
  20. hpcflow/sdk/core/errors.py +245 -61
  21. hpcflow/sdk/core/json_like.py +72 -14
  22. hpcflow/sdk/core/loop.py +122 -21
  23. hpcflow/sdk/core/loop_cache.py +34 -9
  24. hpcflow/sdk/core/object_list.py +172 -26
  25. hpcflow/sdk/core/parallel.py +14 -0
  26. hpcflow/sdk/core/parameters.py +478 -25
  27. hpcflow/sdk/core/rule.py +31 -1
  28. hpcflow/sdk/core/run_dir_files.py +12 -2
  29. hpcflow/sdk/core/task.py +407 -80
  30. hpcflow/sdk/core/task_schema.py +70 -9
  31. hpcflow/sdk/core/test_utils.py +35 -0
  32. hpcflow/sdk/core/utils.py +101 -4
  33. hpcflow/sdk/core/validation.py +13 -1
  34. hpcflow/sdk/core/workflow.py +316 -96
  35. hpcflow/sdk/core/zarr_io.py +23 -0
  36. hpcflow/sdk/data/__init__.py +13 -0
  37. hpcflow/sdk/demo/__init__.py +3 -0
  38. hpcflow/sdk/helper/__init__.py +3 -0
  39. hpcflow/sdk/helper/cli.py +9 -0
  40. hpcflow/sdk/helper/helper.py +28 -0
  41. hpcflow/sdk/helper/watcher.py +33 -0
  42. hpcflow/sdk/log.py +40 -0
  43. hpcflow/sdk/persistence/__init__.py +14 -4
  44. hpcflow/sdk/persistence/base.py +289 -23
  45. hpcflow/sdk/persistence/json.py +29 -0
  46. hpcflow/sdk/persistence/pending.py +217 -107
  47. hpcflow/sdk/persistence/store_resource.py +58 -2
  48. hpcflow/sdk/persistence/utils.py +8 -0
  49. hpcflow/sdk/persistence/zarr.py +68 -1
  50. hpcflow/sdk/runtime.py +52 -10
  51. hpcflow/sdk/submission/__init__.py +3 -0
  52. hpcflow/sdk/submission/jobscript.py +198 -9
  53. hpcflow/sdk/submission/jobscript_info.py +13 -0
  54. hpcflow/sdk/submission/schedulers/__init__.py +60 -0
  55. hpcflow/sdk/submission/schedulers/direct.py +53 -0
  56. hpcflow/sdk/submission/schedulers/sge.py +45 -7
  57. hpcflow/sdk/submission/schedulers/slurm.py +45 -8
  58. hpcflow/sdk/submission/schedulers/utils.py +4 -0
  59. hpcflow/sdk/submission/shells/__init__.py +11 -1
  60. hpcflow/sdk/submission/shells/base.py +32 -1
  61. hpcflow/sdk/submission/shells/bash.py +36 -1
  62. hpcflow/sdk/submission/shells/os_version.py +18 -6
  63. hpcflow/sdk/submission/shells/powershell.py +22 -0
  64. hpcflow/sdk/submission/submission.py +88 -3
  65. hpcflow/sdk/typing.py +10 -1
  66. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/METADATA +1 -1
  67. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/RECORD +70 -70
  68. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/LICENSE +0 -0
  69. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/WHEEL +0 -0
  70. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/entry_points.txt +0 -0
@@ -1,3 +1,9 @@
1
+ """
2
+ Actions are base components of elements.
3
+ Element action runs (EARs) are the basic components of any enactment;
4
+ they may be grouped together within a jobscript for efficiency.
5
+ """
6
+
1
7
  from __future__ import annotations
2
8
  import copy
3
9
  from dataclasses import dataclass
@@ -37,13 +43,23 @@ ACTION_SCOPE_REGEX = r"(\w*)(?:\[(.*)\])?"
37
43
 
38
44
 
39
45
  class ActionScopeType(enum.Enum):
46
+ """
47
+ Types of action scope.
48
+ """
49
+
50
+ #: Scope that applies to anything.
40
51
  ANY = 0
52
+ #: Scope that only applies to main scripts.
41
53
  MAIN = 1
54
+ #: Scope that applies to processing steps.
42
55
  PROCESSING = 2
56
+ #: Scope that applies to input file generators.
43
57
  INPUT_FILE_GENERATOR = 3
58
+ #: Scope that applies to output file parsers.
44
59
  OUTPUT_FILE_PARSER = 4
45
60
 
46
61
 
62
+ #: Keyword arguments permitted for particular scopes.
47
63
  ACTION_SCOPE_ALLOWED_KWARGS = {
48
64
  ActionScopeType.ANY.name: set(),
49
65
  ActionScopeType.MAIN.name: set(),
@@ -64,30 +80,36 @@ class EARStatus(enum.Enum):
64
80
  member.__doc__ = doc
65
81
  return member
66
82
 
83
+ #: Not yet associated with a submission.
67
84
  pending = (
68
85
  0,
69
86
  ".",
70
87
  "grey46",
71
88
  "Not yet associated with a submission.",
72
89
  )
90
+ #: Associated with a prepared submission that is not yet submitted.
73
91
  prepared = (
74
92
  1,
75
93
  ".",
76
94
  "grey46",
77
95
  "Associated with a prepared submission that is not yet submitted.",
78
96
  )
97
+ #: Submitted for execution.
79
98
  submitted = (
80
99
  2,
81
100
  ".",
82
101
  "grey46",
83
102
  "Submitted for execution.",
84
103
  )
104
+ #: Executing now.
85
105
  running = (
86
106
  3,
87
107
  "●",
88
108
  "dodger_blue1",
89
109
  "Executing now.",
90
110
  )
111
+ #: Not attempted due to a failure of an upstream action on which this depends,
112
+ #: or a loop termination condition being satisfied.
91
113
  skipped = (
92
114
  4,
93
115
  "s",
@@ -97,18 +119,21 @@ class EARStatus(enum.Enum):
97
119
  "or a loop termination condition being satisfied."
98
120
  ),
99
121
  )
122
+ #: Aborted by the user; downstream actions will be attempted.
100
123
  aborted = (
101
124
  5,
102
125
  "A",
103
126
  "deep_pink4",
104
127
  "Aborted by the user; downstream actions will be attempted.",
105
128
  )
129
+ #: Probably exited successfully.
106
130
  success = (
107
131
  6,
108
132
  "■",
109
133
  "green3",
110
134
  "Probably exited successfully.",
111
135
  )
136
+ #: Probably failed.
112
137
  error = (
113
138
  7,
114
139
  "E",
@@ -128,10 +153,57 @@ class EARStatus(enum.Enum):
128
153
 
129
154
  @property
130
155
  def rich_repr(self):
156
+ """
157
+ The rich representation of the value.
158
+ """
131
159
  return f"[{self.colour}]{self.symbol}[/{self.colour}]"
132
160
 
133
161
 
134
162
  class ElementActionRun:
163
+ """
164
+ The Element Action Run (EAR) is an atomic unit of an enacted workflow, representing
165
+ one unit of work (e.g., particular submitted job to run a program) within that
166
+ overall workflow. With looping over, say, parameter spaces, there may be many EARs
167
+ per element.
168
+
169
+ Parameters
170
+ ----------
171
+ id_: int
172
+ The ID of the EAR.
173
+ is_pending: bool
174
+ Whether this EAR is pending.
175
+ element_action:
176
+ The particular element action that this is a run of.
177
+ index: int:
178
+ The index of the run within the collection of runs.
179
+ data_idx: dict
180
+ Used for looking up input data to the EAR.
181
+ commands_idx: list[int]
182
+ Indices of commands to apply.
183
+ start_time: datetime
184
+ Time of start of run, if the run has ever been started.
185
+ end_time: datetime
186
+ Time of end of run, if the run has ever ended.
187
+ snapshot_start: dict
188
+ Parameters for taking a snapshot of the data directory before the run.
189
+ If unspecified, no snapshot will be taken.
190
+ snapshot_end: dict
191
+ Parameters for taking a snapshot of the data directory after the run.
192
+ If unspecified, no snapshot will be taken.
193
+ submission_idx: int
194
+ What submission was this (if it has been submitted)?
195
+ success: bool
196
+ Whether this EAR succeeded (if it has run).
197
+ skip: bool
198
+ Whether this EAR was skipped.
199
+ exit_code: int
200
+ The exit code, if known.
201
+ metadata: dict
202
+ Metadata about the EAR.
203
+ run_hostname: str
204
+ Where to run the EAR (if not locally).
205
+ """
206
+
135
207
  _app_attr = "app"
136
208
 
137
209
  def __init__(
@@ -189,14 +261,23 @@ class ElementActionRun:
189
261
 
190
262
  @property
191
263
  def id_(self) -> int:
264
+ """
265
+ The ID of the EAR.
266
+ """
192
267
  return self._id
193
268
 
194
269
  @property
195
270
  def is_pending(self) -> bool:
271
+ """
272
+ Whether this EAR is pending.
273
+ """
196
274
  return self._is_pending
197
275
 
198
276
  @property
199
277
  def element_action(self):
278
+ """
279
+ The particular element action that this is a run of.
280
+ """
200
281
  return self._element_action
201
282
 
202
283
  @property
@@ -206,58 +287,100 @@ class ElementActionRun:
206
287
 
207
288
  @property
208
289
  def action(self):
290
+ """
291
+ The action this is a run of.
292
+ """
209
293
  return self.element_action.action
210
294
 
211
295
  @property
212
296
  def element_iteration(self):
297
+ """
298
+ The iteration information of this run.
299
+ """
213
300
  return self.element_action.element_iteration
214
301
 
215
302
  @property
216
303
  def element(self):
304
+ """
305
+ The element this is a run of.
306
+ """
217
307
  return self.element_iteration.element
218
308
 
219
309
  @property
220
310
  def workflow(self):
311
+ """
312
+ The workflow this is a run of.
313
+ """
221
314
  return self.element_iteration.workflow
222
315
 
223
316
  @property
224
317
  def data_idx(self):
318
+ """
319
+ Used for looking up input data to the EAR.
320
+ """
225
321
  return self._data_idx
226
322
 
227
323
  @property
228
324
  def commands_idx(self):
325
+ """
326
+ Indices of commands to apply.
327
+ """
229
328
  return self._commands_idx
230
329
 
231
330
  @property
232
331
  def metadata(self):
332
+ """
333
+ Metadata about the EAR.
334
+ """
233
335
  return self._metadata
234
336
 
235
337
  @property
236
338
  def run_hostname(self):
339
+ """
340
+ Where to run the EAR, if known/specified.
341
+ """
237
342
  return self._run_hostname
238
343
 
239
344
  @property
240
345
  def start_time(self):
346
+ """
347
+ When the EAR started.
348
+ """
241
349
  return self._start_time
242
350
 
243
351
  @property
244
352
  def end_time(self):
353
+ """
354
+ When the EAR finished.
355
+ """
245
356
  return self._end_time
246
357
 
247
358
  @property
248
359
  def submission_idx(self):
360
+ """
361
+ What actual submission index was this?
362
+ """
249
363
  return self._submission_idx
250
364
 
251
365
  @property
252
366
  def success(self):
367
+ """
368
+ Did the EAR succeed?
369
+ """
253
370
  return self._success
254
371
 
255
372
  @property
256
373
  def skip(self):
374
+ """
375
+ Was the EAR skipped?
376
+ """
257
377
  return self._skip
258
378
 
259
379
  @property
260
380
  def snapshot_start(self):
381
+ """
382
+ The snapshot of the data directory at the start of the run.
383
+ """
261
384
  if self._ss_start_obj is None and self._snapshot_start:
262
385
  self._ss_start_obj = JSONLikeDirSnapShot(
263
386
  root_path=".",
@@ -267,14 +390,18 @@ class ElementActionRun:
267
390
 
268
391
  @property
269
392
  def snapshot_end(self):
393
+ """
394
+ The snapshot of the data directory at the end of the run.
395
+ """
270
396
  if self._ss_end_obj is None and self._snapshot_end:
271
397
  self._ss_end_obj = JSONLikeDirSnapShot(root_path=".", **self._snapshot_end)
272
398
  return self._ss_end_obj
273
399
 
274
400
  @property
275
401
  def dir_diff(self) -> DirectorySnapshotDiff:
276
- """Get the changes to the EAR working directory due to the execution of this
277
- EAR."""
402
+ """
403
+ The changes to the EAR working directory due to the execution of this EAR.
404
+ """
278
405
  if self._ss_diff_obj is None and self.snapshot_end:
279
406
  self._ss_diff_obj = DirectorySnapshotDiff(
280
407
  self.snapshot_start, self.snapshot_end
@@ -283,15 +410,23 @@ class ElementActionRun:
283
410
 
284
411
  @property
285
412
  def exit_code(self):
413
+ """
414
+ The exit code of the underlying program run by the EAR, if known.
415
+ """
286
416
  return self._exit_code
287
417
 
288
418
  @property
289
419
  def task(self):
420
+ """
421
+ The task that this EAR is part of the implementation of.
422
+ """
290
423
  return self.element_action.task
291
424
 
292
425
  @property
293
426
  def status(self):
294
- """Return the state of this EAR."""
427
+ """
428
+ The state of this EAR.
429
+ """
295
430
 
296
431
  if self.skip:
297
432
  return EARStatus.skipped
@@ -336,6 +471,14 @@ class ElementActionRun:
336
471
  return self.action.get_parameter_names(prefix)
337
472
 
338
473
  def get_data_idx(self, path: str = None):
474
+ """
475
+ Get the data index of a value in the most recent iteration.
476
+
477
+ Parameters
478
+ ----------
479
+ path:
480
+ Path to the parameter.
481
+ """
339
482
  return self.element_iteration.get_data_idx(
340
483
  path,
341
484
  action_idx=self.element_action.action_idx,
@@ -350,6 +493,20 @@ class ElementActionRun:
350
493
  as_strings: bool = False,
351
494
  use_task_index: bool = False,
352
495
  ):
496
+ """
497
+ Get the source or sources of a parameter in the most recent iteration.
498
+
499
+ Parameters
500
+ ----------
501
+ path:
502
+ Path to the parameter.
503
+ typ:
504
+ The parameter type.
505
+ as_strings:
506
+ Whether to return the result as human-readable strings.
507
+ use_task_index:
508
+ Whether to use the task index.
509
+ """
353
510
  return self.element_iteration.get_parameter_sources(
354
511
  path,
355
512
  action_idx=self.element_action.action_idx,
@@ -366,6 +523,22 @@ class ElementActionRun:
366
523
  raise_on_missing: bool = False,
367
524
  raise_on_unset: bool = False,
368
525
  ):
526
+ """
527
+ Get a value (parameter, input, output, etc.) from the most recent iteration.
528
+
529
+ Parameters
530
+ ----------
531
+ path:
532
+ Path to the value.
533
+ default:
534
+ Default value to provide if value absent.
535
+ raise_on_missing:
536
+ Whether to raise an exception on an absent value.
537
+ If not, the default is returned.
538
+ raise_on_unset:
539
+ Whether to raise an exception on an explicitly unset value.
540
+ If not, the default is returned.
541
+ """
369
542
  return self.element_iteration.get(
370
543
  path=path,
371
544
  action_idx=self.element_action.action_idx,
@@ -436,12 +609,18 @@ class ElementActionRun:
436
609
 
437
610
  @property
438
611
  def inputs(self):
612
+ """
613
+ The inputs to this EAR.
614
+ """
439
615
  if not self._inputs:
440
616
  self._inputs = self.app.ElementInputs(element_action_run=self)
441
617
  return self._inputs
442
618
 
443
619
  @property
444
620
  def outputs(self):
621
+ """
622
+ The outputs from this EAR.
623
+ """
445
624
  if not self._outputs:
446
625
  self._outputs = self.app.ElementOutputs(element_action_run=self)
447
626
  return self._outputs
@@ -449,24 +628,36 @@ class ElementActionRun:
449
628
  @property
450
629
  @TimeIt.decorator
451
630
  def resources(self):
631
+ """
632
+ The resources to use with (or used by) this EAR.
633
+ """
452
634
  if not self._resources:
453
635
  self._resources = self.app.ElementResources(**self.get_resources())
454
636
  return self._resources
455
637
 
456
638
  @property
457
639
  def input_files(self):
640
+ """
641
+ The input files to the controlled program.
642
+ """
458
643
  if not self._input_files:
459
644
  self._input_files = self.app.ElementInputFiles(element_action_run=self)
460
645
  return self._input_files
461
646
 
462
647
  @property
463
648
  def output_files(self):
649
+ """
650
+ The output files from the controlled program.
651
+ """
464
652
  if not self._output_files:
465
653
  self._output_files = self.app.ElementOutputFiles(element_action_run=self)
466
654
  return self._output_files
467
655
 
468
656
  @property
469
657
  def env_spec(self) -> Dict[str, Any]:
658
+ """
659
+ Environment details.
660
+ """
470
661
  return self.resources.environments[self.action.get_environment_name()]
471
662
 
472
663
  @TimeIt.decorator
@@ -476,9 +667,15 @@ class ElementActionRun:
476
667
  return self.element_iteration.get_resources(self.action)
477
668
 
478
669
  def get_environment_spec(self) -> str:
670
+ """
671
+ What environment to run in?
672
+ """
479
673
  return self.action.get_environment_spec()
480
674
 
481
675
  def get_environment(self) -> app.Environment:
676
+ """
677
+ What environment to run in?
678
+ """
482
679
  return self.action.get_environment()
483
680
 
484
681
  def get_all_previous_iteration_runs(self, include_self: bool = True):
@@ -502,13 +699,13 @@ class ElementActionRun:
502
699
 
503
700
  Parameters
504
701
  ----------
505
- inputs
702
+ inputs:
506
703
  If specified, a list of input parameter types to include, or a dict whose keys
507
704
  are input parameter types to include. For schema inputs that have
508
705
  `multiple=True`, the input type should be labelled. If a dict is passed, and
509
706
  the key "all_iterations` is present and `True`, the return for that input
510
707
  will be structured to include values for all previous iterations.
511
- label_dict
708
+ label_dict:
512
709
  If True, arrange the values of schema inputs with multiple=True as a dict
513
710
  whose keys are the labels. If False, labels will be included in the top level
514
711
  keys.
@@ -565,6 +762,9 @@ class ElementActionRun:
565
762
  return self.get_input_values(inputs=inputs, label_dict=label_dict)
566
763
 
567
764
  def get_IFG_input_values(self) -> Dict[str, Any]:
765
+ """
766
+ Get a dict of input values that are to be passed via an input file generator.
767
+ """
568
768
  if not self.action._from_expand:
569
769
  raise RuntimeError(
570
770
  f"Cannot get input file generator inputs from this EAR because the "
@@ -583,6 +783,10 @@ class ElementActionRun:
583
783
  return inputs
584
784
 
585
785
  def get_OFP_output_files(self) -> Dict[str, Union[str, List[str]]]:
786
+ """
787
+ Get a dict of output files that are going to be parsed to generate one or more
788
+ outputs.
789
+ """
586
790
  # TODO: can this return multiple files for a given FileSpec?
587
791
  if not self.action._from_expand:
588
792
  raise RuntimeError(
@@ -595,6 +799,9 @@ class ElementActionRun:
595
799
  return out_files
596
800
 
597
801
  def get_OFP_inputs(self) -> Dict[str, Union[str, List[str]]]:
802
+ """
803
+ Get a dict of input values that are to be passed to output file parsers.
804
+ """
598
805
  if not self.action._from_expand:
599
806
  raise RuntimeError(
600
807
  f"Cannot get output file parser inputs from this from EAR because the "
@@ -610,6 +817,9 @@ class ElementActionRun:
610
817
  return inputs
611
818
 
612
819
  def get_OFP_outputs(self) -> Dict[str, Union[str, List[str]]]:
820
+ """
821
+ Get the outputs obtained by parsing an output file.
822
+ """
613
823
  if not self.action._from_expand:
614
824
  raise RuntimeError(
615
825
  f"Cannot get output file parser outputs from this from EAR because the "
@@ -621,6 +831,9 @@ class ElementActionRun:
621
831
  return outputs
622
832
 
623
833
  def write_source(self, js_idx: int, js_act_idx: int):
834
+ """
835
+ Write values to files in standard formats.
836
+ """
624
837
  import h5py
625
838
 
626
839
  for fmt, ins in self.action.script_data_in_grouped.items():
@@ -690,10 +903,13 @@ class ElementActionRun:
690
903
  self, jobscript: app.Jobscript, JS_action_idx: int
691
904
  ) -> Tuple[str, List[str], List[int]]:
692
905
  """
906
+ Write the EAR's enactment to disk in preparation for submission.
907
+
693
908
  Returns
694
909
  -------
695
- commands
696
- shell_vars
910
+ commands:
911
+ List of argument words for the command that enacts the EAR.
912
+ shell_vars:
697
913
  Dict whose keys are command indices, and whose values are lists of tuples,
698
914
  where each tuple contains: (parameter name, shell variable name,
699
915
  "stdout"/"stderr").
@@ -735,6 +951,20 @@ class ElementActionRun:
735
951
 
736
952
 
737
953
  class ElementAction:
954
+ """
955
+ An abstract representation of an element's action at a particular iteration and
956
+ the runs that enact that element iteration.
957
+
958
+ Parameters
959
+ ----------
960
+ element_iteration:
961
+ The iteration
962
+ action_idx:
963
+ The action index.
964
+ runs:
965
+ The list of run indices.
966
+ """
967
+
738
968
  _app_attr = "app"
739
969
 
740
970
  def __init__(self, element_iteration, action_idx, runs):
@@ -761,18 +991,30 @@ class ElementAction:
761
991
 
762
992
  @property
763
993
  def element_iteration(self):
994
+ """
995
+ The iteration for this action.
996
+ """
764
997
  return self._element_iteration
765
998
 
766
999
  @property
767
1000
  def element(self):
1001
+ """
1002
+ The element for this action.
1003
+ """
768
1004
  return self.element_iteration.element
769
1005
 
770
1006
  @property
771
1007
  def num_runs(self):
1008
+ """
1009
+ The number of runs associated with this action.
1010
+ """
772
1011
  return len(self._runs)
773
1012
 
774
1013
  @property
775
1014
  def runs(self):
1015
+ """
1016
+ The EARs that this action is enacted by.
1017
+ """
776
1018
  if self._run_objs is None:
777
1019
  self._run_objs = [
778
1020
  self.app.ElementActionRun(
@@ -790,41 +1032,65 @@ class ElementAction:
790
1032
 
791
1033
  @property
792
1034
  def task(self):
1035
+ """
1036
+ The task that this action is an instance of.
1037
+ """
793
1038
  return self.element_iteration.task
794
1039
 
795
1040
  @property
796
1041
  def action_idx(self):
1042
+ """
1043
+ The index of the action.
1044
+ """
797
1045
  return self._action_idx
798
1046
 
799
1047
  @property
800
1048
  def action(self):
1049
+ """
1050
+ The abstract task that this is a concrete model of.
1051
+ """
801
1052
  return self.task.template.get_schema_action(self.action_idx)
802
1053
 
803
1054
  @property
804
1055
  def inputs(self):
1056
+ """
1057
+ The inputs to this action.
1058
+ """
805
1059
  if not self._inputs:
806
1060
  self._inputs = self.app.ElementInputs(element_action=self)
807
1061
  return self._inputs
808
1062
 
809
1063
  @property
810
1064
  def outputs(self):
1065
+ """
1066
+ The outputs from this action.
1067
+ """
811
1068
  if not self._outputs:
812
1069
  self._outputs = self.app.ElementOutputs(element_action=self)
813
1070
  return self._outputs
814
1071
 
815
1072
  @property
816
1073
  def input_files(self):
1074
+ """
1075
+ The input files to this action.
1076
+ """
817
1077
  if not self._input_files:
818
1078
  self._input_files = self.app.ElementInputFiles(element_action=self)
819
1079
  return self._input_files
820
1080
 
821
1081
  @property
822
1082
  def output_files(self):
1083
+ """
1084
+ The output files from this action.
1085
+ """
823
1086
  if not self._output_files:
824
1087
  self._output_files = self.app.ElementOutputFiles(element_action=self)
825
1088
  return self._output_files
826
1089
 
827
1090
  def get_data_idx(self, path: str = None, run_idx: int = -1):
1091
+ """
1092
+ Get the data index for some path/run.
1093
+ """
828
1094
  return self.element_iteration.get_data_idx(
829
1095
  path,
830
1096
  action_idx=self.action_idx,
@@ -839,6 +1105,9 @@ class ElementAction:
839
1105
  as_strings: bool = False,
840
1106
  use_task_index: bool = False,
841
1107
  ):
1108
+ """
1109
+ Get information about where parameters originated.
1110
+ """
842
1111
  return self.element_iteration.get_parameter_sources(
843
1112
  path,
844
1113
  action_idx=self.action_idx,
@@ -856,6 +1125,9 @@ class ElementAction:
856
1125
  raise_on_missing: bool = False,
857
1126
  raise_on_unset: bool = False,
858
1127
  ):
1128
+ """
1129
+ Get the value of a parameter.
1130
+ """
859
1131
  return self.element_iteration.get(
860
1132
  path=path,
861
1133
  action_idx=self.action_idx,
@@ -868,8 +1140,8 @@ class ElementAction:
868
1140
  def get_parameter_names(self, prefix: str) -> List[str]:
869
1141
  """Get parameter types associated with a given prefix.
870
1142
 
871
- For inputs, labels are ignored. See `Action.get_parameter_names` for more
872
- information.
1143
+ For inputs, labels are ignored.
1144
+ See :py:meth:`.Action.get_parameter_names` for more information.
873
1145
 
874
1146
  Parameters
875
1147
  ----------
@@ -898,7 +1170,9 @@ class ActionScope(JSONLike):
898
1170
  if isinstance(typ, str):
899
1171
  typ = getattr(self.app.ActionScopeType, typ.upper())
900
1172
 
1173
+ #: Action scope type.
901
1174
  self.typ = typ
1175
+ #: Any provided extra keyword arguments.
902
1176
  self.kwargs = {k: v for k, v in kwargs.items() if v is not None}
903
1177
 
904
1178
  bad_keys = set(kwargs.keys()) - ACTION_SCOPE_ALLOWED_KWARGS[self.typ.name]
@@ -932,6 +1206,9 @@ class ActionScope(JSONLike):
932
1206
  return {"type": typ_str, **kwargs}
933
1207
 
934
1208
  def to_string(self):
1209
+ """
1210
+ Render this action scope as a string.
1211
+ """
935
1212
  kwargs_str = ""
936
1213
  if self.kwargs:
937
1214
  kwargs_str = "[" + ", ".join(f"{k}={v}" for k, v in self.kwargs.items()) + "]"
@@ -948,27 +1225,46 @@ class ActionScope(JSONLike):
948
1225
 
949
1226
  @classmethod
950
1227
  def any(cls):
1228
+ """
1229
+ Any scope.
1230
+ """
951
1231
  return cls(typ=ActionScopeType.ANY)
952
1232
 
953
1233
  @classmethod
954
1234
  def main(cls):
1235
+ """
1236
+ The main scope.
1237
+ """
955
1238
  return cls(typ=ActionScopeType.MAIN)
956
1239
 
957
1240
  @classmethod
958
1241
  def processing(cls):
1242
+ """
1243
+ The processing scope.
1244
+ """
959
1245
  return cls(typ=ActionScopeType.PROCESSING)
960
1246
 
961
1247
  @classmethod
962
1248
  def input_file_generator(cls, file=None):
1249
+ """
1250
+ The scope of an input file generator.
1251
+ """
963
1252
  return cls(typ=ActionScopeType.INPUT_FILE_GENERATOR, file=file)
964
1253
 
965
1254
  @classmethod
966
1255
  def output_file_parser(cls, output=None):
1256
+ """
1257
+ The scope of an output file parser.
1258
+ """
967
1259
  return cls(typ=ActionScopeType.OUTPUT_FILE_PARSER, output=output)
968
1260
 
969
1261
 
970
1262
  @dataclass
971
1263
  class ActionEnvironment(JSONLike):
1264
+ """
1265
+ The environment that an action is enacted within.
1266
+ """
1267
+
972
1268
  _app_attr = "app"
973
1269
 
974
1270
  _child_objects = (
@@ -978,7 +1274,9 @@ class ActionEnvironment(JSONLike):
978
1274
  ),
979
1275
  )
980
1276
 
1277
+ #: The environment document.
981
1278
  environment: Union[str, Dict[str, Any]]
1279
+ #: The scope.
982
1280
  scope: Optional[app.ActionScope] = None
983
1281
 
984
1282
  def __post_init__(self):
@@ -998,8 +1296,27 @@ class ActionEnvironment(JSONLike):
998
1296
 
999
1297
 
1000
1298
  class ActionRule(JSONLike):
1001
- """Class to represent a rule/condition that must be True if an action is to be
1002
- included."""
1299
+ """
1300
+ Class to represent a rule/condition that must be True if an action is to be
1301
+ included.
1302
+
1303
+ Parameters
1304
+ ----------
1305
+ rule: ~hpcflow.app.Rule
1306
+ The rule to apply.
1307
+ check_exists: str
1308
+ A special rule that is enabled if this named attribute is present.
1309
+ check_missing: str
1310
+ A special rule that is enabled if this named attribute is absent.
1311
+ path: str
1312
+ Where to find the attribute to check.
1313
+ condition: dict | ConditionLike
1314
+ A more complex condition to apply.
1315
+ cast: str
1316
+ The name of a class to cast the attribute to before checking.
1317
+ doc: str
1318
+ Documentation for this rule, if any.
1319
+ """
1003
1320
 
1004
1321
  _child_objects = (ChildObjectSpec(name="rule", class_name="Rule"),)
1005
1322
 
@@ -1031,8 +1348,11 @@ class ActionRule(JSONLike):
1031
1348
  f"constructor arguments."
1032
1349
  )
1033
1350
 
1351
+ #: The rule to apply.
1034
1352
  self.rule = rule
1353
+ #: The action that contains this rule.
1035
1354
  self.action = None # assigned by parent action
1355
+ #: The command that is guarded by this rule.
1036
1356
  self.command = None # assigned by parent command
1037
1357
 
1038
1358
  def __eq__(self, other):
@@ -1044,19 +1364,85 @@ class ActionRule(JSONLike):
1044
1364
 
1045
1365
  @TimeIt.decorator
1046
1366
  def test(self, element_iteration: app.ElementIteration) -> bool:
1367
+ """
1368
+ Test if this rule holds for a particular iteration.
1369
+
1370
+ Parameter
1371
+ ---------
1372
+ element_iteration:
1373
+ The iteration to apply this rule to.
1374
+ """
1047
1375
  return self.rule.test(element_like=element_iteration, action=self.action)
1048
1376
 
1049
1377
  @classmethod
1050
1378
  def check_exists(cls, check_exists):
1379
+ """
1380
+ Make an action rule that checks if a named attribute is present.
1381
+
1382
+ Parameter
1383
+ ---------
1384
+ check_exists: str
1385
+ The path to the attribute to check for.
1386
+ """
1051
1387
  return cls(rule=app.Rule(check_exists=check_exists))
1052
1388
 
1053
1389
  @classmethod
1054
1390
  def check_missing(cls, check_missing):
1391
+ """
1392
+ Make an action rule that checks if a named attribute is absent.
1393
+
1394
+ Parameter
1395
+ ---------
1396
+ check_missing: str
1397
+ The path to the attribute to check for.
1398
+ """
1055
1399
  return cls(rule=app.Rule(check_missing=check_missing))
1056
1400
 
1057
1401
 
1058
1402
  class Action(JSONLike):
1059
- """"""
1403
+ """
1404
+ An atomic component of a workflow that will be enacted within an iteration
1405
+ structure.
1406
+
1407
+ Parameters
1408
+ ----------
1409
+ environments: list[ActionEnvironment]
1410
+ The environments in which this action can run.
1411
+ commands: list[~hpcflow.app.Command]
1412
+ The commands to be run by this action.
1413
+ script: str
1414
+ The name of the Python script to run.
1415
+ script_data_in: str
1416
+ Information about data input to the script.
1417
+ script_data_out: str
1418
+ Information about data output from the script.
1419
+ script_data_files_use_opt: bool
1420
+ If True, script data input and output file paths will be passed to the script
1421
+ execution command line with an option like ``--input-json`` or ``--output-hdf5``
1422
+ etc. If False, the file paths will be passed on their own. For Python scripts,
1423
+ options are always passed, and this parameter is overwritten to be True,
1424
+ regardless of its initial value.
1425
+ script_exe: str
1426
+ The executable to use to run the script.
1427
+ script_pass_env_spec: bool
1428
+ Whether to pass the environment details to the script.
1429
+ abortable: bool
1430
+ Whether this action can be aborted.
1431
+ input_file_generators: list[~hpcflow.app.InputFileGenerator]
1432
+ Any applicable input file generators.
1433
+ output_file_parsers: list[~hpcflow.app.OutputFileParser]
1434
+ Any applicable output file parsers.
1435
+ input_files: list[~hpcflow.app.FileSpec]
1436
+ The input files to the action's commands.
1437
+ output_files: list[~hpcflow.app.FileSpec]
1438
+ The output files from the action's commands.
1439
+ rules: list[ActionRule]
1440
+ How to determine whether to run the action.
1441
+ save_files: list[str]
1442
+ The names of files to be explicitly saved after each step.
1443
+ clean_up: list[str]
1444
+ The names of files to be deleted after each step.
1445
+ """
1060
1446
 
1061
1447
  _app_attr = "app"
1062
1448
  _child_objects = (
@@ -1136,36 +1522,45 @@ class Action(JSONLike):
1136
1522
  save_files: Optional[List[str]] = None,
1137
1523
  clean_up: Optional[List[str]] = None,
1138
1524
  ):
1139
- """
1140
- Parameters
1141
- ----------
1142
- script_data_files_use_opt
1143
- If True, script data input and output file paths will be passed to the script
1144
- execution command line with an option like `--input-json` or `--output-hdf5`
1145
- etc. If False, the file paths will be passed on their own. For Python scripts,
1146
- options are always passed, and this parameter is overwritten to be True,
1147
- regardless of its initial value.
1148
-
1149
- """
1525
+ #: The commands to be run by this action.
1150
1526
  self.commands = commands or []
1527
+ #: The name of the Python script to run.
1151
1528
  self.script = script
1529
+ #: Information about data input to the script.
1152
1530
  self.script_data_in = script_data_in
1531
+ #: Information about data output from the script.
1153
1532
  self.script_data_out = script_data_out
1533
+ #: If True, script data input and output file paths will be passed to the script
1534
+ #: execution command line with an option like `--input-json` or `--output-hdf5`
1535
+ #: etc. If False, the file paths will be passed on their own. For Python scripts,
1536
+ #: options are always passed, and this parameter is overwritten to be True,
1537
+ #: regardless of its initial value.
1154
1538
  self.script_data_files_use_opt = (
1155
1539
  script_data_files_use_opt if not self.script_is_python else True
1156
1540
  )
1541
+ #: The executable to use to run the script.
1157
1542
  self.script_exe = script_exe.lower() if script_exe else None
1543
+ #: Whether to pass the environment details to the script.
1158
1544
  self.script_pass_env_spec = script_pass_env_spec
1545
+ #: The environments in which this action can run.
1159
1546
  self.environments = environments or [
1160
1547
  self.app.ActionEnvironment(environment="null_env")
1161
1548
  ]
1549
+ #: Whether this action can be aborted.
1162
1550
  self.abortable = abortable
1551
+ #: Any applicable input file generators.
1163
1552
  self.input_file_generators = input_file_generators or []
1553
+ #: Any applicable output file parsers.
1164
1554
  self.output_file_parsers = output_file_parsers or []
1555
+ #: The input files to the action's commands.
1165
1556
  self.input_files = self._resolve_input_files(input_files or [])
1557
+ #: The output files from the action's commands.
1166
1558
  self.output_files = self._resolve_output_files(output_files or [])
1559
+ #: How to determine whether to run the action.
1167
1560
  self.rules = rules or []
1561
+ #: The names of files to be explicitly saved after each step.
1168
1562
  self.save_files = save_files or []
1563
+ #: The names of files to be deleted after each step.
1169
1564
  self.clean_up = clean_up or []
1170
1565
 
1171
1566
  self._task_schema = None # assigned by parent TaskSchema
@@ -1174,6 +1569,9 @@ class Action(JSONLike):
1174
1569
  self._set_parent_refs()
1175
1570
 
1176
1571
  def process_script_data_formats(self):
1572
+ """
1573
+ Convert script data information into standard form.
1574
+ """
1177
1575
  self.script_data_in = self._process_script_data_in(self.script_data_in)
1178
1576
  self.script_data_out = self._process_script_data_out(self.script_data_out)
1179
1577
 
@@ -1317,6 +1715,9 @@ class Action(JSONLike):
1317
1715
 
1318
1716
  @property
1319
1717
  def task_schema(self):
1718
+ """
1719
+ The task schema that this action came from.
1720
+ """
1320
1721
  return self._task_schema
1321
1722
 
1322
1723
  def _resolve_input_files(self, input_files):
@@ -1397,7 +1798,7 @@ class Action(JSONLike):
1397
1798
  out = {"input_file_writers": writer_files, "commands": commands}
1398
1799
  return out
1399
1800
 
1400
- def get_resolved_action_env(
1801
+ def _get_resolved_action_env(
1401
1802
  self,
1402
1803
  relevant_scopes: Tuple[app.ActionScopeType],
1403
1804
  input_file_generator: app.InputFileGenerator = None,
@@ -1427,7 +1828,10 @@ class Action(JSONLike):
1427
1828
  def get_input_file_generator_action_env(
1428
1829
  self, input_file_generator: app.InputFileGenerator
1429
1830
  ):
1430
- return self.get_resolved_action_env(
1831
+ """
1832
+ Get the actual environment to use for an input file generator.
1833
+ """
1834
+ return self._get_resolved_action_env(
1431
1835
  relevant_scopes=(
1432
1836
  ActionScopeType.ANY,
1433
1837
  ActionScopeType.PROCESSING,
@@ -1437,7 +1841,10 @@ class Action(JSONLike):
1437
1841
  )
1438
1842
 
1439
1843
  def get_output_file_parser_action_env(self, output_file_parser: app.OutputFileParser):
1440
- return self.get_resolved_action_env(
1844
+ """
1845
+ Get the actual environment to use for an output file parser.
1846
+ """
1847
+ return self._get_resolved_action_env(
1441
1848
  relevant_scopes=(
1442
1849
  ActionScopeType.ANY,
1443
1850
  ActionScopeType.PROCESSING,
@@ -1447,15 +1854,24 @@ class Action(JSONLike):
1447
1854
  )
1448
1855
 
1449
1856
  def get_commands_action_env(self):
1450
- return self.get_resolved_action_env(
1857
+ """
1858
+ Get the actual environment to use for the action commands.
1859
+ """
1860
+ return self._get_resolved_action_env(
1451
1861
  relevant_scopes=(ActionScopeType.ANY, ActionScopeType.MAIN),
1452
1862
  commands=self.commands,
1453
1863
  )
1454
1864
 
1455
1865
  def get_environment_name(self) -> str:
1866
+ """
1867
+ Get the name of the primary environment.
1868
+ """
1456
1869
  return self.get_environment_spec()["name"]
1457
1870
 
1458
1871
  def get_environment_spec(self) -> Dict[str, Any]:
1872
+ """
1873
+ Get the specification for the primary envionment, assuming it has been expanded.
1874
+ """
1459
1875
  if not self._from_expand:
1460
1876
  raise RuntimeError(
1461
1877
  f"Cannot choose a single environment from this action because it is not "
@@ -1464,6 +1880,9 @@ class Action(JSONLike):
1464
1880
  return self.environments[0].environment
1465
1881
 
1466
1882
  def get_environment(self) -> app.Environment:
1883
+ """
1884
+ Get the primary environment.
1885
+ """
1467
1886
  return self.app.envs.get(**self.get_environment_spec())
1468
1887
 
1469
1888
  @staticmethod
@@ -1487,6 +1906,9 @@ class Action(JSONLike):
1487
1906
  def get_snippet_script_str(
1488
1907
  cls, script, env_spec: Optional[Dict[str, Any]] = None
1489
1908
  ) -> str:
1909
+ """
1910
+ Get the substituted script snippet path as a string.
1911
+ """
1490
1912
  if not cls.is_snippet_script(script):
1491
1913
  raise ValueError(
1492
1914
  f"Must be an app-data script name (e.g. "
@@ -1508,6 +1930,9 @@ class Action(JSONLike):
1508
1930
  def get_snippet_script_path(
1509
1931
  cls, script_path, env_spec: Optional[Dict[str, Any]] = None
1510
1932
  ) -> Path:
1933
+ """
1934
+ Get the substituted script snippet path, or False if there is no snippet.
1935
+ """
1511
1936
  if not cls.is_snippet_script(script_path):
1512
1937
  return False
1513
1938
 
@@ -1518,26 +1943,42 @@ class Action(JSONLike):
1518
1943
  return Path(path)
1519
1944
 
1520
1945
  @staticmethod
1521
- def get_param_dump_file_stem(js_idx: int, js_act_idx: int):
1946
+ def __get_param_dump_file_stem(js_idx: int, js_act_idx: int):
1522
1947
  return RunDirAppFiles.get_run_param_dump_file_prefix(js_idx, js_act_idx)
1523
1948
 
1524
1949
  @staticmethod
1525
- def get_param_load_file_stem(js_idx: int, js_act_idx: int):
1950
+ def __get_param_load_file_stem(js_idx: int, js_act_idx: int):
1526
1951
  return RunDirAppFiles.get_run_param_load_file_prefix(js_idx, js_act_idx)
1527
1952
 
1528
1953
  def get_param_dump_file_path_JSON(self, js_idx: int, js_act_idx: int):
1529
- return Path(self.get_param_dump_file_stem(js_idx, js_act_idx) + ".json")
1954
+ """
1955
+ Get the path of the JSON dump file.
1956
+ """
1957
+ return Path(self.__get_param_dump_file_stem(js_idx, js_act_idx) + ".json")
1530
1958
 
1531
1959
  def get_param_dump_file_path_HDF5(self, js_idx: int, js_act_idx: int):
1532
- return Path(self.get_param_dump_file_stem(js_idx, js_act_idx) + ".h5")
1960
+ """
1961
+ Get the path of the HDF56 dump file.
1962
+ """
1963
+ return Path(self.__get_param_dump_file_stem(js_idx, js_act_idx) + ".h5")
1533
1964
 
1534
1965
  def get_param_load_file_path_JSON(self, js_idx: int, js_act_idx: int):
1535
- return Path(self.get_param_load_file_stem(js_idx, js_act_idx) + ".json")
1966
+ """
1967
+ Get the path of the JSON load file.
1968
+ """
1969
+ return Path(self.__get_param_load_file_stem(js_idx, js_act_idx) + ".json")
1536
1970
 
1537
1971
  def get_param_load_file_path_HDF5(self, js_idx: int, js_act_idx: int):
1538
- return Path(self.get_param_load_file_stem(js_idx, js_act_idx) + ".h5")
1972
+ """
1973
+ Get the path of the HDF5 load file.
1974
+ """
1975
+ return Path(self.__get_param_load_file_stem(js_idx, js_act_idx) + ".h5")
1539
1976
 
1540
1977
  def expand(self):
1978
+ """
1979
+ Expand this action into a list of actions if necessary.
1980
+ This converts input file generators and output file parsers into their own actions.
1981
+ """
1541
1982
  if self._from_expand:
1542
1983
  # already expanded
1543
1984
  return [self]
@@ -1694,7 +2135,7 @@ class Action(JSONLike):
1694
2135
 
1695
2136
  Parameters
1696
2137
  ----------
1697
- sub_parameters
2138
+ sub_parameters:
1698
2139
  If True, sub-parameters (i.e. dot-delimited parameter types) will be returned
1699
2140
  untouched. If False (default), only return the root parameter type and
1700
2141
  disregard the sub-parameter part.
@@ -1747,7 +2188,7 @@ class Action(JSONLike):
1747
2188
 
1748
2189
  Parameters
1749
2190
  ----------
1750
- sub_parameters
2191
+ sub_parameters:
1751
2192
  If True, sub-parameters (i.e. dot-delimited parameter types) in command line
1752
2193
  inputs will be returned untouched. If False (default), only return the root
1753
2194
  parameter type and disregard the sub-parameter part.
@@ -1786,9 +2227,15 @@ class Action(JSONLike):
1786
2227
  return tuple(set(params))
1787
2228
 
1788
2229
  def get_input_file_labels(self):
2230
+ """
2231
+ Get the labels from the input files.
2232
+ """
1789
2233
  return tuple(i.label for i in self.input_files)
1790
2234
 
1791
2235
  def get_output_file_labels(self):
2236
+ """
2237
+ Get the labels from the output files.
2238
+ """
1792
2239
  return tuple(i.label for i in self.output_files)
1793
2240
 
1794
2241
  @TimeIt.decorator
@@ -1919,6 +2366,10 @@ class Action(JSONLike):
1919
2366
  return scopes
1920
2367
 
1921
2368
  def get_precise_scope(self) -> app.ActionScope:
2369
+ """
2370
+ Get the exact scope of this action.
2371
+ The action must have been expanded prior to calling this.
2372
+ """
1922
2373
  if not self._from_expand:
1923
2374
  raise RuntimeError(
1924
2375
  "Precise scope cannot be unambiguously defined until the Action has been "
@@ -1942,6 +2393,9 @@ class Action(JSONLike):
1942
2393
  def is_input_type_required(
1943
2394
  self, typ: str, provided_files: List[app.FileSpec]
1944
2395
  ) -> bool:
2396
+ """
2397
+ Determine if the given input type is required by this action.
2398
+ """
1945
2399
  # TODO: for now assume a script takes all inputs
1946
2400
  if (
1947
2401
  self.script
@@ -1966,6 +2420,9 @@ class Action(JSONLike):
1966
2420
  if typ in (OFP.inputs or []):
1967
2421
  return True
1968
2422
 
2423
+ # Appears to be not required
2424
+ return False
2425
+
1969
2426
  @TimeIt.decorator
1970
2427
  def test_rules(self, element_iter) -> Tuple[bool, List[int]]:
1971
2428
  """Test all rules against the specified element iteration."""