hpcflow-new2 0.2.0a178__py3-none-any.whl → 0.2.0a180__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. hpcflow/_version.py +1 -1
  2. hpcflow/data/demo_data_manifest/__init__.py +3 -0
  3. hpcflow/sdk/__init__.py +4 -1
  4. hpcflow/sdk/app.py +160 -15
  5. hpcflow/sdk/cli.py +14 -0
  6. hpcflow/sdk/cli_common.py +83 -0
  7. hpcflow/sdk/config/__init__.py +4 -0
  8. hpcflow/sdk/config/callbacks.py +25 -2
  9. hpcflow/sdk/config/cli.py +4 -1
  10. hpcflow/sdk/config/config.py +188 -14
  11. hpcflow/sdk/config/config_file.py +91 -3
  12. hpcflow/sdk/config/errors.py +33 -0
  13. hpcflow/sdk/core/__init__.py +2 -0
  14. hpcflow/sdk/core/actions.py +492 -35
  15. hpcflow/sdk/core/cache.py +22 -0
  16. hpcflow/sdk/core/command_files.py +221 -5
  17. hpcflow/sdk/core/commands.py +57 -0
  18. hpcflow/sdk/core/element.py +407 -8
  19. hpcflow/sdk/core/environment.py +92 -0
  20. hpcflow/sdk/core/errors.py +245 -61
  21. hpcflow/sdk/core/json_like.py +72 -14
  22. hpcflow/sdk/core/loop.py +122 -21
  23. hpcflow/sdk/core/loop_cache.py +34 -9
  24. hpcflow/sdk/core/object_list.py +172 -26
  25. hpcflow/sdk/core/parallel.py +14 -0
  26. hpcflow/sdk/core/parameters.py +478 -25
  27. hpcflow/sdk/core/rule.py +31 -1
  28. hpcflow/sdk/core/run_dir_files.py +12 -2
  29. hpcflow/sdk/core/task.py +407 -80
  30. hpcflow/sdk/core/task_schema.py +70 -9
  31. hpcflow/sdk/core/test_utils.py +35 -0
  32. hpcflow/sdk/core/utils.py +101 -4
  33. hpcflow/sdk/core/validation.py +13 -1
  34. hpcflow/sdk/core/workflow.py +316 -96
  35. hpcflow/sdk/core/zarr_io.py +23 -0
  36. hpcflow/sdk/data/__init__.py +13 -0
  37. hpcflow/sdk/demo/__init__.py +3 -0
  38. hpcflow/sdk/helper/__init__.py +3 -0
  39. hpcflow/sdk/helper/cli.py +9 -0
  40. hpcflow/sdk/helper/helper.py +28 -0
  41. hpcflow/sdk/helper/watcher.py +33 -0
  42. hpcflow/sdk/log.py +40 -0
  43. hpcflow/sdk/persistence/__init__.py +14 -4
  44. hpcflow/sdk/persistence/base.py +289 -23
  45. hpcflow/sdk/persistence/json.py +29 -0
  46. hpcflow/sdk/persistence/pending.py +217 -107
  47. hpcflow/sdk/persistence/store_resource.py +58 -2
  48. hpcflow/sdk/persistence/utils.py +8 -0
  49. hpcflow/sdk/persistence/zarr.py +68 -1
  50. hpcflow/sdk/runtime.py +52 -10
  51. hpcflow/sdk/submission/__init__.py +3 -0
  52. hpcflow/sdk/submission/jobscript.py +198 -9
  53. hpcflow/sdk/submission/jobscript_info.py +13 -0
  54. hpcflow/sdk/submission/schedulers/__init__.py +60 -0
  55. hpcflow/sdk/submission/schedulers/direct.py +53 -0
  56. hpcflow/sdk/submission/schedulers/sge.py +45 -7
  57. hpcflow/sdk/submission/schedulers/slurm.py +45 -8
  58. hpcflow/sdk/submission/schedulers/utils.py +4 -0
  59. hpcflow/sdk/submission/shells/__init__.py +11 -1
  60. hpcflow/sdk/submission/shells/base.py +32 -1
  61. hpcflow/sdk/submission/shells/bash.py +36 -1
  62. hpcflow/sdk/submission/shells/os_version.py +18 -6
  63. hpcflow/sdk/submission/shells/powershell.py +22 -0
  64. hpcflow/sdk/submission/submission.py +88 -3
  65. hpcflow/sdk/typing.py +10 -1
  66. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/METADATA +1 -1
  67. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/RECORD +70 -70
  68. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/LICENSE +0 -0
  69. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/WHEEL +0 -0
  70. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/core/task.py CHANGED
@@ -1,3 +1,7 @@
1
+ """
2
+ Tasks are components of workflows.
3
+ """
4
+
1
5
  from __future__ import annotations
2
6
  from collections import defaultdict
3
7
  import copy
@@ -66,18 +70,59 @@ class InputStatus:
66
70
 
67
71
  """
68
72
 
73
+ #: True if a default value is available.
69
74
  has_default: bool
75
+ #: True if the input is required by one or more actions. An input may not be required
76
+ #: if it is only used in the generation of inputs files, and those input files are
77
+ #: passed to the element set directly.
70
78
  is_required: bool
79
+ #: True if the input is locally provided in the element set.
71
80
  is_provided: bool
72
81
 
73
82
  @property
74
83
  def is_extra(self):
75
- """Return True if the input is provided but not required."""
84
+ """True if the input is provided but not required."""
76
85
  return self.is_provided and not self.is_required
77
86
 
78
87
 
79
88
  class ElementSet(JSONLike):
80
- """Class to represent a parametrisation of a new set of elements."""
89
+ """Class to represent a parameterisation of a new set of elements.
90
+
91
+ Parameters
92
+ ----------
93
+ inputs: list[~hpcflow.app.InputValue]
94
+ Inputs to the set of elements.
95
+ input_files: list[~hpcflow.app.InputFile]
96
+ Input files to the set of elements.
97
+ sequences: list[~hpcflow.app.ValueSequence]
98
+ Input value sequences to parameterise over.
99
+ resources: ~hpcflow.app.ResourceList
100
+ Resources to use for the set of elements.
101
+ repeats: list[dict]
102
+ Description of how to repeat the set of elements.
103
+ groups: list[~hpcflow.app.ElementGroup]
104
+ Groupings in the set of elements.
105
+ input_sources: dict[str, ~hpcflow.app.InputSource]
106
+ Input source descriptors.
107
+ nesting_order: dict[str, int]
108
+ How to handle nesting of iterations.
109
+ env_preset: str
110
+ Which environment preset to use. Don't use at same time as ``environments``.
111
+ environments: dict
112
+ Environment descriptors to use. Don't use at same time as ``env_preset``.
113
+ sourceable_elem_iters: list[int]
114
+ If specified, a list of global element iteration indices from which inputs for
115
+ the new elements associated with this element set may be sourced. If not
116
+ specified, all workflow element iterations are considered sourceable.
117
+ allow_non_coincident_task_sources: bool
118
+ If True, if more than one parameter is sourced from the same task, then allow
119
+ these sources to come from distinct element sub-sets. If False (default),
120
+ only the intersection of element sub-sets for all parameters are included.
121
+ merge_envs: bool
122
+ If True, merge ``environments`` into ``resources`` using the "any" scope. If
123
+ False, ``environments`` are ignored. This is required on first initialisation,
124
+ but not on subsequent re-initialisation from a persistent workflow.
125
+ """
81
126
 
82
127
  _child_objects = (
83
128
  ChildObjectSpec(
@@ -137,35 +182,34 @@ class ElementSet(JSONLike):
137
182
  allow_non_coincident_task_sources: Optional[bool] = False,
138
183
  merge_envs: Optional[bool] = True,
139
184
  ):
140
- """
141
- Parameters
142
- ----------
143
- sourceable_elem_iters
144
- If specified, a list of global element iteration indices from which inputs for
145
- the new elements associated with this element set may be sourced. If not
146
- specified, all workflow element iterations are considered sourceable.
147
- allow_non_coincident_task_sources
148
- If True, if more than one parameter is sourced from the same task, then allow
149
- these sources to come from distinct element sub-sets. If False (default),
150
- only the intersection of element sub-sets for all parameters are included.
151
- merge_envs
152
- If True, merge `environments` into `resources` using the "any" scope. If
153
- False, `environments` are ignored. This is required on first initialisation,
154
- but not on subsequent re-initialisation from a persistent workflow.
155
- """
156
-
185
+ #: Inputs to the set of elements.
157
186
  self.inputs = inputs or []
187
+ #: Input files to the set of elements.
158
188
  self.input_files = input_files or []
189
+ #: Description of how to repeat the set of elements.
159
190
  self.repeats = repeats or []
191
+ #: Groupings in the set of elements.
160
192
  self.groups = groups or []
193
+ #: Resources to use for the set of elements.
161
194
  self.resources = self.app.ResourceList.normalise(resources)
195
+ #: Input value sequences to parameterise over.
162
196
  self.sequences = sequences or []
197
+ #: Input source descriptors.
163
198
  self.input_sources = input_sources or {}
199
+ #: How to handle nesting of iterations.
164
200
  self.nesting_order = nesting_order or {}
201
+ #: Which environment preset to use.
165
202
  self.env_preset = env_preset
203
+ #: Environment descriptors to use.
166
204
  self.environments = environments
205
+ #: List of global element iteration indices from which inputs for
206
+ #: the new elements associated with this element set may be sourced.
207
+ #: If ``None``, all iterations are valid.
167
208
  self.sourceable_elem_iters = sourceable_elem_iters
209
+ #: Whether to allow sources to come from distinct element sub-sets.
168
210
  self.allow_non_coincident_task_sources = allow_non_coincident_task_sources
211
+ #: Whether to merge ``environments`` into ``resources`` using the "any" scope
212
+ #: on first initialisation.
169
213
  self.merge_envs = merge_envs
170
214
 
171
215
  self._validate()
@@ -235,6 +279,9 @@ class ElementSet(JSONLike):
235
279
 
236
280
  @property
237
281
  def task_template(self):
282
+ """
283
+ The abstract task this was derived from.
284
+ """
238
285
  return self._task_template
239
286
 
240
287
  @task_template.setter
@@ -244,11 +291,14 @@ class ElementSet(JSONLike):
244
291
 
245
292
  @property
246
293
  def input_types(self):
294
+ """
295
+ The input types of the inputs to this element set.
296
+ """
247
297
  return [i.labelled_type for i in self.inputs]
248
298
 
249
299
  @property
250
300
  def element_local_idx_range(self):
251
- """Used to retrieve elements belonging to this element set."""
301
+ """Indices of elements belonging to this element set."""
252
302
  return tuple(self._element_local_idx_range)
253
303
 
254
304
  def _validate(self):
@@ -376,6 +426,9 @@ class ElementSet(JSONLike):
376
426
  element_sets=None,
377
427
  sourceable_elem_iters=None,
378
428
  ):
429
+ """
430
+ Make an instance after validating some argument combinations.
431
+ """
379
432
  args = (
380
433
  inputs,
381
434
  input_files,
@@ -417,18 +470,30 @@ class ElementSet(JSONLike):
417
470
 
418
471
  @property
419
472
  def defined_input_types(self):
473
+ """
474
+ The input types to this element set.
475
+ """
420
476
  return self._defined_input_types
421
477
 
422
478
  @property
423
479
  def undefined_input_types(self):
480
+ """
481
+ The input types to the abstract task that aren't related to this element set.
482
+ """
424
483
  return self.task_template.all_schema_input_types - self.defined_input_types
425
484
 
426
485
  def get_sequence_from_path(self, sequence_path):
427
- for i in self.sequences:
428
- if i.path == sequence_path:
429
- return i
486
+ """
487
+ Get the value sequence for the given path, if it exists.
488
+ """
489
+ for seq in self.sequences:
490
+ if seq.path == sequence_path:
491
+ return seq
430
492
 
431
493
  def get_defined_parameter_types(self):
494
+ """
495
+ Get the parameter types of this element set.
496
+ """
432
497
  out = []
433
498
  for inp in self.inputs:
434
499
  if not inp.is_sub_value:
@@ -439,6 +504,9 @@ class ElementSet(JSONLike):
439
504
  return out
440
505
 
441
506
  def get_defined_sub_parameter_types(self):
507
+ """
508
+ Get the sub-parameter types of this element set.
509
+ """
442
510
  out = []
443
511
  for inp in self.inputs:
444
512
  if inp.is_sub_value:
@@ -449,35 +517,48 @@ class ElementSet(JSONLike):
449
517
  return out
450
518
 
451
519
  def get_locally_defined_inputs(self):
520
+ """
521
+ Get the input types that this element set defines.
522
+ """
452
523
  return self.get_defined_parameter_types() + self.get_defined_sub_parameter_types()
453
524
 
454
- def get_sequence_by_path(self, path):
455
- for seq in self.sequences:
456
- if seq.path == path:
457
- return seq
458
-
459
525
  @property
460
526
  def index(self):
527
+ """
528
+ The index of this element set in its' template task's collection of sets.
529
+ """
461
530
  for idx, element_set in enumerate(self.task_template.element_sets):
462
531
  if element_set is self:
463
532
  return idx
464
533
 
465
534
  @property
466
535
  def task(self):
536
+ """
537
+ The concrete task corresponding to this element set.
538
+ """
467
539
  return self.task_template.workflow_template.workflow.tasks[
468
540
  self.task_template.index
469
541
  ]
470
542
 
471
543
  @property
472
544
  def elements(self):
545
+ """
546
+ The elements in this element set.
547
+ """
473
548
  return self.task.elements[slice(*self.element_local_idx_range)]
474
549
 
475
550
  @property
476
551
  def element_iterations(self):
552
+ """
553
+ The iterations in this element set.
554
+ """
477
555
  return [j for i in self.elements for j in i.iterations]
478
556
 
479
557
  @property
480
558
  def elem_iter_IDs(self):
559
+ """
560
+ The IDs of the iterations in this element set.
561
+ """
481
562
  return [i.id_ for i in self.element_iterations]
482
563
 
483
564
  def get_task_dependencies(self, as_objects=False):
@@ -510,8 +591,18 @@ class ElementSet(JSONLike):
510
591
 
511
592
 
512
593
  class OutputLabel(JSONLike):
513
- """Class to represent schema input labels that should be applied to a subset of task
514
- outputs"""
594
+ """
595
+ Schema input labels that should be applied to a subset of task outputs.
596
+
597
+ Parameters
598
+ ----------
599
+ parameter:
600
+ Name of a parameter.
601
+ label:
602
+ Label to apply to the parameter.
603
+ where: ~hpcflow.app.ElementFilter
604
+ Optional filtering rule
605
+ """
515
606
 
516
607
  _child_objects = (
517
608
  ChildObjectSpec(
@@ -526,22 +617,50 @@ class OutputLabel(JSONLike):
526
617
  label: str,
527
618
  where: Optional[List[app.ElementFilter]] = None,
528
619
  ) -> None:
620
+ #: Name of a parameter.
529
621
  self.parameter = parameter
622
+ #: Label to apply to the parameter.
530
623
  self.label = label
624
+ #: Filtering rule.
531
625
  self.where = where
532
626
 
533
627
 
534
628
  class Task(JSONLike):
535
- """Parametrisation of an isolated task for which a subset of input values are given
629
+ """
630
+ Parametrisation of an isolated task for which a subset of input values are given
536
631
  "locally". The remaining input values are expected to be satisfied by other
537
632
  tasks/imports in the workflow.
538
633
 
539
634
  Parameters
540
635
  ----------
541
- schema
542
- A `TaskSchema` object or a list of `TaskSchema` objects.
543
- inputs
636
+ schema: ~hpcflow.app.TaskSchema | list[~hpcflow.app.TaskSchema]
637
+ A (list of) `TaskSchema` object(s) and/or a (list of) strings that are task
638
+ schema names that uniquely identify a task schema. If strings are provided,
639
+ the `TaskSchema` object will be fetched from the known task schemas loaded by
640
+ the app configuration.
641
+ repeats: list[dict]
642
+ groups: list[~hpcflow.app.ElementGroup]
643
+ resources: dict
644
+ inputs: list[~hpcflow.app.InputValue]
544
645
  A list of `InputValue` objects.
646
+ input_files: list[~hpcflow.app.InputFile]
647
+ sequences: list[~hpcflow.app.ValueSequence]
648
+ input_sources: dict[str, ~hpcflow.app.InputSource]
649
+ nesting_order: list
650
+ env_preset: str
651
+ environments: dict[str, dict]
652
+ allow_non_coincident_task_sources: bool
653
+ If True, if more than one parameter is sourced from the same task, then allow
654
+ these sources to come from distinct element sub-sets. If False (default),
655
+ only the intersection of element sub-sets for all parameters are included.
656
+ element_sets: list[ElementSet]
657
+ output_labels: list[OutputLabel]
658
+ sourceable_elem_iters: list[int]
659
+ merge_envs: bool
660
+ If True, merge environment presets (set via the element set `env_preset` key)
661
+ into `resources` using the "any" scope. If False, these presets are ignored.
662
+ This is required on first initialisation, but not on subsequent
663
+ re-initialisation from a persistent workflow.
545
664
  """
546
665
 
547
666
  _child_objects = (
@@ -585,25 +704,6 @@ class Task(JSONLike):
585
704
  sourceable_elem_iters: Optional[List[int]] = None,
586
705
  merge_envs: Optional[bool] = True,
587
706
  ):
588
- """
589
- Parameters
590
- ----------
591
- schema
592
- A (list of) `TaskSchema` object(s) and/or a (list of) strings that are task
593
- schema names that uniquely identify a task schema. If strings are provided,
594
- the `TaskSchema` object will be fetched from the known task schemas loaded by
595
- the app configuration.
596
- allow_non_coincident_task_sources
597
- If True, if more than one parameter is sourced from the same task, then allow
598
- these sources to come from distinct element sub-sets. If False (default),
599
- only the intersection of element sub-sets for all parameters are included.
600
- merge_envs
601
- If True, merge environment presets (set via the element set `env_preset` key)
602
- into `resources` using the "any" scope. If False, these presets are ignored.
603
- This is required on first initialisation, but not on subsequent
604
- re-initialisation from a persistent workflow.
605
- """
606
-
607
707
  # TODO: allow init via specifying objective and/or method and/or implementation
608
708
  # (lists of) strs e.g.: Task(
609
709
  # objective='simulate_VE_loading',
@@ -652,6 +752,8 @@ class Task(JSONLike):
652
752
  sourceable_elem_iters=sourceable_elem_iters,
653
753
  )
654
754
  self._output_labels = output_labels or []
755
+ #: Whether to merge ``environments`` into ``resources`` using the "any" scope
756
+ #: on first initialisation.
655
757
  self.merge_envs = merge_envs
656
758
 
657
759
  # appended to when new element sets are added and reset on dump to disk:
@@ -660,6 +762,7 @@ class Task(JSONLike):
660
762
  self._validate()
661
763
  self._name = self._get_name()
662
764
 
765
+ #: The template workflow that this task is within.
663
766
  self.workflow_template = None # assigned by parent WorkflowTemplate
664
767
  self._insert_ID = None
665
768
  self._dir_name = None
@@ -800,6 +903,9 @@ class Task(JSONLike):
800
903
  }
801
904
 
802
905
  def set_sequence_parameters(self, element_set):
906
+ """
907
+ Set up parameters parsed by value sequences.
908
+ """
803
909
  # set ValueSequence Parameter objects:
804
910
  for seq in element_set.sequences:
805
911
  if seq.input_type:
@@ -882,6 +988,11 @@ class Task(JSONLike):
882
988
  return output_data_indices
883
989
 
884
990
  def prepare_element_resolution(self, element_set, input_data_indices):
991
+ """
992
+ Set up the resolution of details of elements
993
+ (especially multiplicities and how iterations are nested)
994
+ within an element set.
995
+ """
885
996
  multiplicities = []
886
997
  for path_i, inp_idx_i in input_data_indices.items():
887
998
  multiplicities.append(
@@ -917,6 +1028,9 @@ class Task(JSONLike):
917
1028
 
918
1029
  @property
919
1030
  def index(self):
1031
+ """
1032
+ The index of this task within the workflow's tasks.
1033
+ """
920
1034
  if self.workflow_template:
921
1035
  return self.workflow_template.tasks.index(self)
922
1036
  else:
@@ -924,6 +1038,9 @@ class Task(JSONLike):
924
1038
 
925
1039
  @property
926
1040
  def output_labels(self):
1041
+ """
1042
+ The labels on the outputs of the task.
1043
+ """
927
1044
  return self._output_labels
928
1045
 
929
1046
  @property
@@ -1113,11 +1230,14 @@ class Task(JSONLike):
1113
1230
 
1114
1231
  @property
1115
1232
  def schemas(self) -> List[app.TaskSchema]:
1233
+ """
1234
+ All the task schemas.
1235
+ """
1116
1236
  return self._schemas
1117
1237
 
1118
1238
  @property
1119
1239
  def schema(self) -> app.TaskSchema:
1120
- """Returns the single task schema, if only one, else raises."""
1240
+ """The single task schema, if only one, else raises."""
1121
1241
  if len(self._schemas) == 1:
1122
1242
  return self._schemas[0]
1123
1243
  else:
@@ -1128,52 +1248,81 @@ class Task(JSONLike):
1128
1248
 
1129
1249
  @property
1130
1250
  def element_sets(self):
1251
+ """
1252
+ The element sets.
1253
+ """
1131
1254
  return self._element_sets + self._pending_element_sets
1132
1255
 
1133
1256
  @property
1134
1257
  def num_element_sets(self):
1258
+ """
1259
+ The number of element sets.
1260
+ """
1135
1261
  return len(self.element_sets)
1136
1262
 
1137
1263
  @property
1138
1264
  def insert_ID(self):
1265
+ """
1266
+ Insertion ID.
1267
+ """
1139
1268
  return self._insert_ID
1140
1269
 
1141
1270
  @property
1142
1271
  def dir_name(self):
1143
- "Artefact directory name."
1272
+ """
1273
+ Artefact directory name.
1274
+ """
1144
1275
  return self._dir_name
1145
1276
 
1146
1277
  @property
1147
1278
  def name(self):
1279
+ """
1280
+ Task name.
1281
+ """
1148
1282
  return self._name
1149
1283
 
1150
1284
  @property
1151
1285
  def objective(self):
1286
+ """
1287
+ The goal of this task.
1288
+ """
1152
1289
  return self.schemas[0].objective
1153
1290
 
1154
1291
  @property
1155
1292
  def all_schema_inputs(self) -> Tuple[app.SchemaInput]:
1293
+ """
1294
+ The inputs to this task's schemas.
1295
+ """
1156
1296
  return tuple(inp_j for schema_i in self.schemas for inp_j in schema_i.inputs)
1157
1297
 
1158
1298
  @property
1159
1299
  def all_schema_outputs(self) -> Tuple[app.SchemaOutput]:
1300
+ """
1301
+ The outputs from this task's schemas.
1302
+ """
1160
1303
  return tuple(inp_j for schema_i in self.schemas for inp_j in schema_i.outputs)
1161
1304
 
1162
1305
  @property
1163
1306
  def all_schema_input_types(self):
1164
- """Get the set of all schema input types (over all specified schemas)."""
1307
+ """The set of all schema input types (over all specified schemas)."""
1165
1308
  return {inp_j for schema_i in self.schemas for inp_j in schema_i.input_types}
1166
1309
 
1167
1310
  @property
1168
1311
  def all_schema_input_normalised_paths(self):
1312
+ """
1313
+ Normalised paths for all schema input types.
1314
+ """
1169
1315
  return {f"inputs.{i}" for i in self.all_schema_input_types}
1170
1316
 
1171
1317
  @property
1172
1318
  def all_schema_output_types(self):
1173
- """Get the set of all schema output types (over all specified schemas)."""
1319
+ """The set of all schema output types (over all specified schemas)."""
1174
1320
  return {out_j for schema_i in self.schemas for out_j in schema_i.output_types}
1175
1321
 
1176
1322
  def get_schema_action(self, idx):
1323
+ """
1324
+ Get the schema action at the given index.
1325
+ """
1177
1326
  _idx = 0
1178
1327
  for schema in self.schemas:
1179
1328
  for action in schema.actions:
@@ -1183,6 +1332,9 @@ class Task(JSONLike):
1183
1332
  raise ValueError(f"No action in task {self.name!r} with index {idx!r}.")
1184
1333
 
1185
1334
  def all_schema_actions(self) -> Iterator[Tuple[int, app.Action]]:
1335
+ """
1336
+ Get all the schema actions and their indices.
1337
+ """
1186
1338
  idx = 0
1187
1339
  for schema in self.schemas:
1188
1340
  for action in schema.actions:
@@ -1191,6 +1343,9 @@ class Task(JSONLike):
1191
1343
 
1192
1344
  @property
1193
1345
  def num_all_schema_actions(self) -> int:
1346
+ """
1347
+ The total number of schema actions.
1348
+ """
1194
1349
  num = 0
1195
1350
  for schema in self.schemas:
1196
1351
  for _ in schema.actions:
@@ -1199,6 +1354,9 @@ class Task(JSONLike):
1199
1354
 
1200
1355
  @property
1201
1356
  def all_sourced_normalised_paths(self):
1357
+ """
1358
+ All the sourced normalised paths, including of sub-values.
1359
+ """
1202
1360
  sourced_input_types = []
1203
1361
  for elem_set in self.element_sets:
1204
1362
  for inp in elem_set.inputs:
@@ -1280,14 +1438,23 @@ class Task(JSONLike):
1280
1438
 
1281
1439
  @property
1282
1440
  def defined_input_types(self):
1441
+ """
1442
+ The input types defined by this task.
1443
+ """
1283
1444
  return self._defined_input_types
1284
1445
 
1285
1446
  @property
1286
1447
  def undefined_input_types(self):
1448
+ """
1449
+ The schema's input types that this task doesn't define.
1450
+ """
1287
1451
  return self.all_schema_input_types - self.defined_input_types
1288
1452
 
1289
1453
  @property
1290
1454
  def undefined_inputs(self):
1455
+ """
1456
+ The task's inputs that are undefined.
1457
+ """
1291
1458
  return [
1292
1459
  inp_j
1293
1460
  for schema_i in self.schemas
@@ -1323,6 +1490,9 @@ class Task(JSONLike):
1323
1490
  def add_group(
1324
1491
  self, name: str, where: app.ElementFilter, group_by_distinct: app.ParameterPath
1325
1492
  ):
1493
+ """
1494
+ Add an element group to this task.
1495
+ """
1326
1496
  group = ElementGroup(name=name, where=where, group_by_distinct=group_by_distinct)
1327
1497
  self.groups.add_object(group)
1328
1498
 
@@ -1346,7 +1516,20 @@ class Task(JSONLike):
1346
1516
 
1347
1517
 
1348
1518
  class WorkflowTask:
1349
- """Class to represent a Task that is bound to a Workflow."""
1519
+ """
1520
+ Represents a :py:class:`Task` that is bound to a :py:class:`Workflow`.
1521
+
1522
+ Parameters
1523
+ ----------
1524
+ workflow:
1525
+ The workflow that the task is bound to.
1526
+ template:
1527
+ The task template that this binds.
1528
+ index:
1529
+ Where in the workflow's list of tasks is this one.
1530
+ element_IDs:
1531
+ The IDs of the elements of this task.
1532
+ """
1350
1533
 
1351
1534
  _app_attr = "app"
1352
1535
 
@@ -1379,6 +1562,18 @@ class WorkflowTask:
1379
1562
 
1380
1563
  @classmethod
1381
1564
  def new_empty_task(cls, workflow: app.Workflow, template: app.Task, index: int):
1565
+ """
1566
+ Make a new instance without any elements set up yet.
1567
+
1568
+ Parameters
1569
+ ----------
1570
+ workflow:
1571
+ The workflow that the task is bound to.
1572
+ template:
1573
+ The task template that this binds.
1574
+ index:
1575
+ Where in the workflow's list of tasks is this one.
1576
+ """
1382
1577
  obj = cls(
1383
1578
  workflow=workflow,
1384
1579
  template=template,
@@ -1389,61 +1584,103 @@ class WorkflowTask:
1389
1584
 
1390
1585
  @property
1391
1586
  def workflow(self):
1587
+ """
1588
+ The workflow this task is bound to.
1589
+ """
1392
1590
  return self._workflow
1393
1591
 
1394
1592
  @property
1395
1593
  def template(self):
1594
+ """
1595
+ The template for this task.
1596
+ """
1396
1597
  return self._template
1397
1598
 
1398
1599
  @property
1399
1600
  def index(self):
1601
+ """
1602
+ The index of this task within its workflow.
1603
+ """
1400
1604
  return self._index
1401
1605
 
1402
1606
  @property
1403
1607
  def element_IDs(self):
1608
+ """
1609
+ The IDs of elements associated with this task.
1610
+ """
1404
1611
  return self._element_IDs + self._pending_element_IDs
1405
1612
 
1406
1613
  @property
1407
1614
  def num_elements(self):
1615
+ """
1616
+ The number of elements associated with this task.
1617
+ """
1408
1618
  return len(self.element_IDs)
1409
1619
 
1410
1620
  @property
1411
1621
  def num_actions(self):
1622
+ """
1623
+ The number of actions in this task.
1624
+ """
1412
1625
  return self.template.num_all_schema_actions
1413
1626
 
1414
1627
  @property
1415
1628
  def name(self):
1629
+ """
1630
+ The name of this task based on its template.
1631
+ """
1416
1632
  return self.template.name
1417
1633
 
1418
1634
  @property
1419
1635
  def unique_name(self):
1636
+ """
1637
+ The unique name for this task specifically.
1638
+ """
1420
1639
  return self.workflow.get_task_unique_names()[self.index]
1421
1640
 
1422
1641
  @property
1423
1642
  def insert_ID(self):
1643
+ """
1644
+ The insertion ID of the template task.
1645
+ """
1424
1646
  return self.template.insert_ID
1425
1647
 
1426
1648
  @property
1427
1649
  def dir_name(self):
1650
+ """
1651
+ The name of the directory for the task's temporary files.
1652
+ """
1428
1653
  return self.template.dir_name
1429
1654
 
1430
1655
  @property
1431
1656
  def num_element_sets(self):
1657
+ """
1658
+ The number of element sets associated with this task.
1659
+ """
1432
1660
  return self.template.num_element_sets
1433
1661
 
1434
1662
  @property
1435
1663
  @TimeIt.decorator
1436
1664
  def elements(self):
1665
+ """
1666
+ The elements associated with this task.
1667
+ """
1437
1668
  if self._elements is None:
1438
1669
  self._elements = self.app.Elements(self)
1439
1670
  return self._elements
1440
1671
 
1441
1672
  def get_dir_name(self, loop_idx: Dict[str, int] = None) -> str:
1673
+ """
1674
+ Get the directory name for a particular iteration.
1675
+ """
1442
1676
  if not loop_idx:
1443
1677
  return self.dir_name
1444
1678
  return self.dir_name + "_" + "_".join((f"{k}-{v}" for k, v in loop_idx.items()))
1445
1679
 
1446
1680
  def get_all_element_iterations(self) -> Dict[int, app.ElementIteration]:
1681
+ """
1682
+ Get the iterations known by the task's elements.
1683
+ """
1447
1684
  return {j.id_: j for i in self.elements for j in i.iterations}
1448
1685
 
1449
1686
  def _make_new_elements_persistent(
@@ -1633,7 +1870,7 @@ class WorkflowTask:
1633
1870
  source_idx[key] = [inp_src_idx] * len(grp_idx)
1634
1871
  if key in sequence_idx:
1635
1872
  sequence_idx.pop(key)
1636
- seq = element_set.get_sequence_by_path(key)
1873
+ seq = element_set.get_sequence_from_path(key)
1637
1874
 
1638
1875
  elif inp_src.source_type is InputSourceType.DEFAULT:
1639
1876
  grp_idx = [def_val._value_group_idx]
@@ -1950,6 +2187,9 @@ class WorkflowTask:
1950
2187
  sequence_indices,
1951
2188
  source_indices,
1952
2189
  ):
2190
+ """
2191
+ Create information about new elements in this task.
2192
+ """
1953
2193
  new_elements = []
1954
2194
  element_sequence_indices = {}
1955
2195
  element_src_indices = {}
@@ -1984,12 +2224,12 @@ class WorkflowTask:
1984
2224
 
1985
2225
  @property
1986
2226
  def upstream_tasks(self):
1987
- """Get all workflow tasks that are upstream from this task."""
2227
+ """All workflow tasks that are upstream from this task."""
1988
2228
  return [task for task in self.workflow.tasks[: self.index]]
1989
2229
 
1990
2230
  @property
1991
2231
  def downstream_tasks(self):
1992
- """Get all workflow tasks that are downstream from this task."""
2232
+ """All workflow tasks that are downstream from this task."""
1993
2233
  return [task for task in self.workflow.tasks[self.index + 1 :]]
1994
2234
 
1995
2235
  @staticmethod
@@ -2246,6 +2486,22 @@ class WorkflowTask:
2246
2486
  propagate_to=None,
2247
2487
  return_indices=False,
2248
2488
  ):
2489
+ """
2490
+ Add elements to this task.
2491
+
2492
+ Parameters
2493
+ ----------
2494
+ sourceable_elem_iters : list of int, optional
2495
+ If specified, a list of global element iteration indices from which inputs
2496
+ may be sourced. If not specified, all workflow element iterations are
2497
+ considered sourceable.
2498
+ propagate_to : dict[str, ElementPropagation]
2499
+ Propagate the new elements downstream to the specified tasks.
2500
+ return_indices : bool
2501
+ If True, return the list of indices of the newly added elements. False by
2502
+ default.
2503
+
2504
+ """
2249
2505
  propagate_to = self.app.ElementPropagation._prepare_propagate_to_dict(
2250
2506
  propagate_to, self.workflow
2251
2507
  )
@@ -2281,21 +2537,7 @@ class WorkflowTask:
2281
2537
  propagate_to: Dict[str, app.ElementPropagation] = None,
2282
2538
  return_indices: bool = False,
2283
2539
  ):
2284
- """Add more elements to this task.
2285
-
2286
- Parameters
2287
- ----------
2288
- sourceable_elem_iters : list of int, optional
2289
- If specified, a list of global element iteration indices from which inputs
2290
- may be sourced. If not specified, all workflow element iterations are
2291
- considered sourceable.
2292
- propagate_to : dict of [str, ElementPropagation]
2293
- Propagate the new elements downstream to the specified tasks.
2294
- return_indices : bool, optional
2295
- If True, return the list of indices of the newly added elements. False by
2296
- default.
2297
-
2298
- """
2540
+ """Add more elements to this task."""
2299
2541
 
2300
2542
  if base_element is not None:
2301
2543
  if base_element.task is not self:
@@ -2467,13 +2709,22 @@ class WorkflowTask:
2467
2709
 
2468
2710
  @property
2469
2711
  def inputs(self):
2712
+ """
2713
+ Inputs to this task.
2714
+ """
2470
2715
  return self.app.TaskInputParameters(self)
2471
2716
 
2472
2717
  @property
2473
2718
  def outputs(self):
2719
+ """
2720
+ Outputs from this task.
2721
+ """
2474
2722
  return self.app.TaskOutputParameters(self)
2475
2723
 
2476
2724
  def get(self, path, raise_on_missing=False, default=None):
2725
+ """
2726
+ Get a parameter known to this task by its path.
2727
+ """
2477
2728
  return self.app.Parameters(
2478
2729
  self,
2479
2730
  path=path,
@@ -2866,6 +3117,15 @@ class WorkflowTask:
2866
3117
 
2867
3118
 
2868
3119
  class Elements:
3120
+ """
3121
+ The elements of a task. Iterable.
3122
+
3123
+ Parameters
3124
+ ----------
3125
+ task:
3126
+ The task this will be the elements of.
3127
+ """
3128
+
2869
3129
  __slots__ = ("_task",)
2870
3130
 
2871
3131
  def __init__(self, task: app.WorkflowTask):
@@ -2881,6 +3141,9 @@ class Elements:
2881
3141
 
2882
3142
  @property
2883
3143
  def task(self):
3144
+ """
3145
+ The task this is the elements of.
3146
+ """
2884
3147
  return self._task
2885
3148
 
2886
3149
  @TimeIt.decorator
@@ -2924,13 +3187,38 @@ class Elements:
2924
3187
 
2925
3188
  @dataclass
2926
3189
  class Parameters:
3190
+ """
3191
+ The parameters of a (workflow-bound) task. Iterable.
3192
+
3193
+ Parameters
3194
+ ----------
3195
+ task: WorkflowTask
3196
+ The task these are the parameters of.
3197
+ path: str
3198
+ The path to the parameter or parameters.
3199
+ return_element_parameters: bool
3200
+ Whether to return element parameters.
3201
+ raise_on_missing: bool
3202
+ Whether to raise an exception on a missing parameter.
3203
+ raise_on_unset: bool
3204
+ Whether to raise an exception on an unset parameter.
3205
+ default:
3206
+ A default value to use when the parameter is absent.
3207
+ """
3208
+
2927
3209
  _app_attr = "_app"
2928
3210
 
3211
+ #: The task these are the parameters of.
2929
3212
  task: app.WorkflowTask
3213
+ #: The path to the parameter or parameters.
2930
3214
  path: str
3215
+ #: Whether to return element parameters.
2931
3216
  return_element_parameters: bool
3217
+ #: Whether to raise an exception on a missing parameter.
2932
3218
  raise_on_missing: Optional[bool] = False
3219
+ #: Whether to raise an exception on an unset parameter.
2933
3220
  raise_on_unset: Optional[bool] = False
3221
+ #: A default value to use when the parameter is absent.
2934
3222
  default: Optional[Any] = None
2935
3223
 
2936
3224
  @TimeIt.decorator
@@ -2990,10 +3278,19 @@ class Parameters:
2990
3278
 
2991
3279
  @dataclass
2992
3280
  class TaskInputParameters:
2993
- """For retrieving schema input parameters across all elements."""
3281
+ """
3282
+ For retrieving schema input parameters across all elements.
3283
+ Treat as an unmodifiable namespace.
3284
+
3285
+ Parameters
3286
+ ----------
3287
+ task:
3288
+ The task that this represents the input parameters of.
3289
+ """
2994
3290
 
2995
3291
  _app_attr = "_app"
2996
3292
 
3293
+ #: The task that this represents the input parameters of.
2997
3294
  task: app.WorkflowTask
2998
3295
 
2999
3296
  def __getattr__(self, name):
@@ -3016,10 +3313,19 @@ class TaskInputParameters:
3016
3313
 
3017
3314
  @dataclass
3018
3315
  class TaskOutputParameters:
3019
- """For retrieving schema output parameters across all elements."""
3316
+ """
3317
+ For retrieving schema output parameters across all elements.
3318
+ Treat as an unmodifiable namespace.
3319
+
3320
+ Parameters
3321
+ ----------
3322
+ task:
3323
+ The task that this represents the output parameters of.
3324
+ """
3020
3325
 
3021
3326
  _app_attr = "_app"
3022
3327
 
3328
+ #: The task that this represents the output parameters of.
3023
3329
  task: app.WorkflowTask
3024
3330
 
3025
3331
  def __getattr__(self, name):
@@ -3042,17 +3348,38 @@ class TaskOutputParameters:
3042
3348
 
3043
3349
  @dataclass
3044
3350
  class ElementPropagation:
3045
- """Class to represent how a newly added element set should propagate to a given
3046
- downstream task."""
3351
+ """
3352
+ Class to represent how a newly added element set should propagate to a given
3353
+ downstream task.
3354
+
3355
+ Parameters
3356
+ ----------
3357
+ task:
3358
+ The task this is propagating to.
3359
+ nesting_order:
3360
+ The nesting order information.
3361
+ input_sources:
3362
+ The input source information.
3363
+ """
3047
3364
 
3048
3365
  _app_attr = "app"
3049
3366
 
3367
+ #: The task this is propagating to.
3050
3368
  task: app.Task
3369
+ #: The nesting order information.
3051
3370
  nesting_order: Optional[Dict] = None
3371
+ #: The input source information.
3052
3372
  input_sources: Optional[Dict] = None
3053
3373
 
3054
3374
  @property
3055
3375
  def element_set(self):
3376
+ """
3377
+ The element set that this propagates from.
3378
+
3379
+ Note
3380
+ ----
3381
+ Temporary property. May be moved or reinterpreted.
3382
+ """
3056
3383
  # TEMP property; for now just use the first element set as the base:
3057
3384
  return self.task.template.element_sets[0]
3058
3385