hpcflow-new2 0.2.0a178__py3-none-any.whl → 0.2.0a180__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. hpcflow/_version.py +1 -1
  2. hpcflow/data/demo_data_manifest/__init__.py +3 -0
  3. hpcflow/sdk/__init__.py +4 -1
  4. hpcflow/sdk/app.py +160 -15
  5. hpcflow/sdk/cli.py +14 -0
  6. hpcflow/sdk/cli_common.py +83 -0
  7. hpcflow/sdk/config/__init__.py +4 -0
  8. hpcflow/sdk/config/callbacks.py +25 -2
  9. hpcflow/sdk/config/cli.py +4 -1
  10. hpcflow/sdk/config/config.py +188 -14
  11. hpcflow/sdk/config/config_file.py +91 -3
  12. hpcflow/sdk/config/errors.py +33 -0
  13. hpcflow/sdk/core/__init__.py +2 -0
  14. hpcflow/sdk/core/actions.py +492 -35
  15. hpcflow/sdk/core/cache.py +22 -0
  16. hpcflow/sdk/core/command_files.py +221 -5
  17. hpcflow/sdk/core/commands.py +57 -0
  18. hpcflow/sdk/core/element.py +407 -8
  19. hpcflow/sdk/core/environment.py +92 -0
  20. hpcflow/sdk/core/errors.py +245 -61
  21. hpcflow/sdk/core/json_like.py +72 -14
  22. hpcflow/sdk/core/loop.py +122 -21
  23. hpcflow/sdk/core/loop_cache.py +34 -9
  24. hpcflow/sdk/core/object_list.py +172 -26
  25. hpcflow/sdk/core/parallel.py +14 -0
  26. hpcflow/sdk/core/parameters.py +478 -25
  27. hpcflow/sdk/core/rule.py +31 -1
  28. hpcflow/sdk/core/run_dir_files.py +12 -2
  29. hpcflow/sdk/core/task.py +407 -80
  30. hpcflow/sdk/core/task_schema.py +70 -9
  31. hpcflow/sdk/core/test_utils.py +35 -0
  32. hpcflow/sdk/core/utils.py +101 -4
  33. hpcflow/sdk/core/validation.py +13 -1
  34. hpcflow/sdk/core/workflow.py +316 -96
  35. hpcflow/sdk/core/zarr_io.py +23 -0
  36. hpcflow/sdk/data/__init__.py +13 -0
  37. hpcflow/sdk/demo/__init__.py +3 -0
  38. hpcflow/sdk/helper/__init__.py +3 -0
  39. hpcflow/sdk/helper/cli.py +9 -0
  40. hpcflow/sdk/helper/helper.py +28 -0
  41. hpcflow/sdk/helper/watcher.py +33 -0
  42. hpcflow/sdk/log.py +40 -0
  43. hpcflow/sdk/persistence/__init__.py +14 -4
  44. hpcflow/sdk/persistence/base.py +289 -23
  45. hpcflow/sdk/persistence/json.py +29 -0
  46. hpcflow/sdk/persistence/pending.py +217 -107
  47. hpcflow/sdk/persistence/store_resource.py +58 -2
  48. hpcflow/sdk/persistence/utils.py +8 -0
  49. hpcflow/sdk/persistence/zarr.py +68 -1
  50. hpcflow/sdk/runtime.py +52 -10
  51. hpcflow/sdk/submission/__init__.py +3 -0
  52. hpcflow/sdk/submission/jobscript.py +198 -9
  53. hpcflow/sdk/submission/jobscript_info.py +13 -0
  54. hpcflow/sdk/submission/schedulers/__init__.py +60 -0
  55. hpcflow/sdk/submission/schedulers/direct.py +53 -0
  56. hpcflow/sdk/submission/schedulers/sge.py +45 -7
  57. hpcflow/sdk/submission/schedulers/slurm.py +45 -8
  58. hpcflow/sdk/submission/schedulers/utils.py +4 -0
  59. hpcflow/sdk/submission/shells/__init__.py +11 -1
  60. hpcflow/sdk/submission/shells/base.py +32 -1
  61. hpcflow/sdk/submission/shells/bash.py +36 -1
  62. hpcflow/sdk/submission/shells/os_version.py +18 -6
  63. hpcflow/sdk/submission/shells/powershell.py +22 -0
  64. hpcflow/sdk/submission/submission.py +88 -3
  65. hpcflow/sdk/typing.py +10 -1
  66. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/METADATA +1 -1
  67. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/RECORD +70 -70
  68. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/LICENSE +0 -0
  69. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/WHEEL +0 -0
  70. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/entry_points.txt +0 -0
@@ -1,3 +1,7 @@
1
+ """
2
+ Parameters represent information passed around within a workflow.
3
+ """
4
+
1
5
  from __future__ import annotations
2
6
  import copy
3
7
  from dataclasses import dataclass, field
@@ -49,40 +53,74 @@ def _process_demo_data_strings(app, value):
49
53
 
50
54
 
51
55
  class ParameterValue:
56
+ """
57
+ The value handler for a parameter.
58
+
59
+ Intended to be subclassed.
60
+ """
61
+
52
62
  _typ = None
53
63
  _sub_parameters = {}
54
64
 
55
65
  def to_dict(self):
66
+ """
67
+ Serialise this parameter value as a dictionary.
68
+ """
56
69
  if hasattr(self, "__dict__"):
57
70
  return dict(self.__dict__)
58
71
  elif hasattr(self, "__slots__"):
59
72
  return {k: getattr(self, k) for k in self.__slots__}
60
73
 
61
74
  def prepare_JSON_dump(self) -> Dict:
75
+ """
76
+ Prepare this parameter value for serialisation as JSON.
77
+ """
62
78
  raise NotImplementedError
63
79
 
64
80
  def dump_to_HDF5_group(self, group):
81
+ """
82
+ Write this parameter value to an HDF5 group.
83
+ """
65
84
  raise NotImplementedError
66
85
 
67
86
  @classmethod
68
87
  def save_from_HDF5_group(cls, group, param_id: int, workflow):
88
+ """
89
+ Extract a parameter value from an HDF5 group.
90
+ """
69
91
  raise NotImplementedError
70
92
 
71
93
  @classmethod
72
94
  def save_from_JSON(cls, data, param_id: int, workflow):
95
+ """
96
+ Extract a parameter value from JSON data.
97
+ """
73
98
  raise NotImplementedError
74
99
 
75
100
 
76
101
  class ParameterPropagationMode(enum.Enum):
102
+ """
103
+ How a parameter is propagated.
104
+ """
105
+
106
+ #: Parameter is propagated implicitly.
77
107
  IMPLICIT = 0
108
+ #: Parameter is propagated explicitly.
78
109
  EXPLICIT = 1
110
+ #: Parameter is never propagated.
79
111
  NEVER = 2
80
112
 
81
113
 
82
114
  @dataclass
83
115
  class ParameterPath(JSONLike):
116
+ """
117
+ Path to a parameter.
118
+ """
119
+
84
120
  # TODO: unused?
121
+ #: The path to the parameter.
85
122
  path: Sequence[Union[str, int, float]]
123
+ #: The task in which to look up the parameter.
86
124
  task: Optional[
87
125
  Union[app.TaskTemplate, app.TaskSchema]
88
126
  ] = None # default is "current" task
@@ -90,6 +128,28 @@ class ParameterPath(JSONLike):
90
128
 
91
129
  @dataclass
92
130
  class Parameter(JSONLike):
131
+ """
132
+ A general parameter to a workflow task.
133
+
134
+ Parameters
135
+ ----------
136
+ typ:
137
+ Type code.
138
+ Used to look up the :py:class:`ParameterValue` for this parameter,
139
+ if any.
140
+ is_file:
141
+ Whether this parameter represents a file.
142
+ sub_parameters: list[SubParameter]
143
+ Any parameters packed within this one.
144
+ _value_class: type[ParameterValue]
145
+ Class that provides the implementation of this parameter's values.
146
+ Not normally directly user-managed.
147
+ _hash_value:
148
+ Hash of this class. Not normally user-managed.
149
+ _validation:
150
+ Validation schema.
151
+ """
152
+
93
153
  _validation_schema = "parameters_spec_schema.yaml"
94
154
  _child_objects = (
95
155
  ChildObjectSpec(
@@ -102,8 +162,12 @@ class Parameter(JSONLike):
102
162
  ),
103
163
  )
104
164
 
165
+ #: Type code. Used to look up the :py:class:`ParameterValue` for this parameter,
166
+ #: if any.
105
167
  typ: str
168
+ #: Whether this parameter represents a file.
106
169
  is_file: bool = False
170
+ #: Any parameters packed within this one.
107
171
  sub_parameters: List[app.SubParameter] = field(default_factory=lambda: [])
108
172
  _value_class: Any = None
109
173
  _hash_value: Optional[str] = field(default=None, repr=False)
@@ -158,17 +222,35 @@ class Parameter(JSONLike):
158
222
 
159
223
  @property
160
224
  def url_slug(self) -> str:
225
+ """
226
+ Representation of this parameter as part of a URL.
227
+ """
161
228
  return self.typ.lower().replace("_", "-")
162
229
 
163
230
 
164
231
  @dataclass
165
232
  class SubParameter:
233
+ """
234
+ A parameter that is a component of another parameter.
235
+ """
236
+
237
+ #: How to find this within the containing paraneter.
166
238
  address: Address
239
+ #: The containing main parameter.
167
240
  parameter: app.Parameter
168
241
 
169
242
 
170
243
  @dataclass
171
244
  class SchemaParameter(JSONLike):
245
+ """
246
+ A parameter bound in a schema.
247
+
248
+ Parameters
249
+ ----------
250
+ parameter: Parameter
251
+ The parameter.
252
+ """
253
+
172
254
  _app_attr = "app"
173
255
 
174
256
  _child_objects = (
@@ -189,14 +271,26 @@ class SchemaParameter(JSONLike):
189
271
 
190
272
  @property
191
273
  def name(self):
274
+ """
275
+ The name of the parameter.
276
+ """
192
277
  return self.parameter.name
193
278
 
194
279
  @property
195
280
  def typ(self):
281
+ """
282
+ The type code of the parameter.
283
+ """
196
284
  return self.parameter.typ
197
285
 
198
286
 
199
287
  class NullDefault(enum.Enum):
288
+ """
289
+ Sentinel value used to distinguish an explicit null.
290
+ """
291
+
292
+ #: Special sentinel.
293
+ #: Used in situations where otherwise a JSON object or array would be.
200
294
  NULL = 0
201
295
 
202
296
 
@@ -206,13 +300,13 @@ class SchemaInput(SchemaParameter):
206
300
 
207
301
  Parameters
208
302
  ----------
209
- parameter
303
+ parameter:
210
304
  The parameter (i.e. type) of this schema input.
211
- multiple
305
+ multiple:
212
306
  If True, expect one or more of these parameters defined in the workflow,
213
307
  distinguished by a string label in square brackets. For example `p1[0]` for a
214
308
  parameter `p1`.
215
- labels
309
+ labels:
216
310
  Dict whose keys represent the string labels that distinguish multiple parameters
217
311
  if `multiple` is `True`. Use the key "*" to mean all labels not matching
218
312
  other label keys. If `multiple` is `False`, this will default to a
@@ -220,10 +314,10 @@ class SchemaInput(SchemaParameter):
220
314
  `True`, this will default to a single-item dict with the catch-all key:
221
315
  `{{"*": {{}}}}`. On initialisation, remaining keyword-arguments are treated as default
222
316
  values for the dict values of `labels`.
223
- default_value
317
+ default_value:
224
318
  The default value for this input parameter. This is itself a default value that
225
319
  will be applied to all `labels` values if a "default_value" key does not exist.
226
- propagation_mode
320
+ propagation_mode:
227
321
  Determines how this input should propagate through the workflow. This is a default
228
322
  value that will be applied to all `labels` values if a "propagation_mode" key does
229
323
  not exist. By default, the input is allowed to be used in downstream tasks simply
@@ -232,7 +326,7 @@ class SchemaInput(SchemaParameter):
232
326
  the downstream task `input_sources` for it to be used, and "never", meaning that
233
327
  the parameter must not be used in downstream tasks and will be inaccessible to
234
328
  those tasks.
235
- group
329
+ group:
236
330
  Determines the name of the element group from which this input should be sourced.
237
331
  This is a default value that will be applied to all `labels` if a "group" key
238
332
  does not exist.
@@ -270,8 +364,12 @@ class SchemaInput(SchemaParameter):
270
364
  except ValueError:
271
365
  parameter = self.app.Parameter(parameter)
272
366
 
367
+ #: The parameter (i.e. type) of this schema input.
273
368
  self.parameter = parameter
369
+ #: Whether to expect more than of these parameters defined in the workflow.
274
370
  self.multiple = multiple
371
+ #: Dict whose keys represent the string labels that distinguish multiple
372
+ #: parameters if `multiple` is `True`.
275
373
  self.labels = labels
276
374
 
277
375
  if self.labels is None:
@@ -395,6 +493,9 @@ class SchemaInput(SchemaParameter):
395
493
 
396
494
  @property
397
495
  def default_value(self):
496
+ """
497
+ The default value of the input.
498
+ """
398
499
  if not self.multiple:
399
500
  if "default_value" in self.single_labelled_data:
400
501
  return self.single_labelled_data["default_value"]
@@ -403,28 +504,46 @@ class SchemaInput(SchemaParameter):
403
504
 
404
505
  @property
405
506
  def task_schema(self):
507
+ """
508
+ The schema containing this input.
509
+ """
406
510
  return self._task_schema
407
511
 
408
512
  @property
409
513
  def all_labelled_types(self):
514
+ """
515
+ The types of the input labels.
516
+ """
410
517
  return list(f"{self.typ}{f'[{i}]' if i else ''}" for i in self.labels)
411
518
 
412
519
  @property
413
520
  def single_label(self):
521
+ """
522
+ The label of this input, assuming it is not mulitple.
523
+ """
414
524
  if not self.multiple:
415
525
  return next(iter(self.labels))
416
526
 
417
527
  @property
418
528
  def single_labelled_type(self):
529
+ """
530
+ The type code of this input, assuming it is not mulitple.
531
+ """
419
532
  if not self.multiple:
420
533
  return next(iter(self.labelled_info()))["labelled_type"]
421
534
 
422
535
  @property
423
536
  def single_labelled_data(self):
537
+ """
538
+ The value of this input, assuming it is not mulitple.
539
+ """
424
540
  if not self.multiple:
425
541
  return self.labels[self.single_label]
426
542
 
427
543
  def labelled_info(self):
544
+ """
545
+ Get descriptors for all the labels associated with this input.
546
+ """
428
547
  for k, v in self.labels.items():
429
548
  label = f"[{k}]" if k else ""
430
549
  dct = {
@@ -458,6 +577,9 @@ class SchemaInput(SchemaParameter):
458
577
 
459
578
  @property
460
579
  def input_or_output(self):
580
+ """
581
+ Whether this is an input or output. Always ``input``.
582
+ """
461
583
  return "input"
462
584
 
463
585
 
@@ -465,11 +587,16 @@ class SchemaInput(SchemaParameter):
465
587
  class SchemaOutput(SchemaParameter):
466
588
  """A Parameter as outputted from particular task."""
467
589
 
590
+ #: The basic parameter this supplies.
468
591
  parameter: Parameter
592
+ #: How this output propagates.
469
593
  propagation_mode: ParameterPropagationMode = ParameterPropagationMode.IMPLICIT
470
594
 
471
595
  @property
472
596
  def input_or_output(self):
597
+ """
598
+ Whether this is an input or output. Always ``output``.
599
+ """
473
600
  return "output"
474
601
 
475
602
  def __repr__(self) -> str:
@@ -483,6 +610,11 @@ class SchemaOutput(SchemaParameter):
483
610
 
484
611
  @dataclass
485
612
  class BuiltinSchemaParameter:
613
+ """
614
+ A parameter of a built-in schema.
615
+ """
616
+
617
+ # TODO: Is this used anywhere?
486
618
  # builtin inputs (resources,parameter_perturbations,method,implementation
487
619
  # builtin outputs (time, memory use, node/hostname etc)
488
620
  # - builtin parameters do not propagate to other tasks (since all tasks define the same
@@ -493,6 +625,23 @@ class BuiltinSchemaParameter:
493
625
 
494
626
 
495
627
  class ValueSequence(JSONLike):
628
+ """
629
+ A sequence of values.
630
+
631
+ Parameters
632
+ ----------
633
+ path:
634
+ The path to this sequence.
635
+ values:
636
+ The values in this sequence.
637
+ nesting_order: int
638
+ A nesting order for this sequence. Can be used to compose sequences together.
639
+ label: str
640
+ A label for this sequence.
641
+ value_class_method: str
642
+ Name of a method used to generate sequence values. Not normally used directly.
643
+ """
644
+
496
645
  def __init__(
497
646
  self,
498
647
  path: str,
@@ -504,9 +653,13 @@ class ValueSequence(JSONLike):
504
653
  label = str(label) if label is not None else ""
505
654
  path, label = self._validate_parameter_path(path, label)
506
655
 
656
+ #: The path to this sequence.
507
657
  self.path = path
658
+ #: The label of this sequence.
508
659
  self.label = label
660
+ #: The nesting order for this sequence.
509
661
  self.nesting_order = nesting_order
662
+ #: Name of a method used to generate sequence values.
510
663
  self.value_class_method = value_class_method
511
664
 
512
665
  if values is not None:
@@ -603,30 +756,48 @@ class ValueSequence(JSONLike):
603
756
 
604
757
  @property
605
758
  def parameter(self):
759
+ """
760
+ The parameter this sequence supplies.
761
+ """
606
762
  return self._parameter
607
763
 
608
764
  @property
609
765
  def path_split(self):
766
+ """
767
+ The components of ths path.
768
+ """
610
769
  if self._path_split is None:
611
770
  self._path_split = self.path.split(".")
612
771
  return self._path_split
613
772
 
614
773
  @property
615
774
  def path_type(self):
775
+ """
776
+ The type of path this is.
777
+ """
616
778
  return self.path_split[0]
617
779
 
618
780
  @property
619
781
  def input_type(self):
782
+ """
783
+ The type of input sequence this is, if it is one.
784
+ """
620
785
  if self.path_type == "inputs":
621
786
  return self.path_split[1].replace(self._label_fmt, "")
622
787
 
623
788
  @property
624
789
  def input_path(self):
790
+ """
791
+ The path of the input sequence this is, if it is one.
792
+ """
625
793
  if self.path_type == "inputs":
626
794
  return ".".join(self.path_split[2:])
627
795
 
628
796
  @property
629
797
  def resource_scope(self):
798
+ """
799
+ The scope of the resources this is, if it is one.
800
+ """
630
801
  if self.path_type == "resources":
631
802
  return self.path_split[1]
632
803
 
@@ -641,6 +812,9 @@ class ValueSequence(JSONLike):
641
812
 
642
813
  @property
643
814
  def labelled_type(self):
815
+ """
816
+ The labelled type of input sequence this is, if it is one.
817
+ """
644
818
  if self.input_type:
645
819
  return f"{self.input_type}{self._label_fmt}"
646
820
 
@@ -750,12 +924,17 @@ class ValueSequence(JSONLike):
750
924
 
751
925
  @property
752
926
  def normalised_path(self):
927
+ """
928
+ The path to this sequence.
929
+ """
753
930
  return self.path
754
931
 
755
932
  @property
756
933
  def normalised_inputs_path(self):
757
- """Return the normalised path without the "inputs" prefix, if the sequence is an
758
- inputs sequence, else return None."""
934
+ """
935
+ The normalised path without the "inputs" prefix, if the sequence is an
936
+ inputs sequence, else return None.
937
+ """
759
938
 
760
939
  if self.input_type:
761
940
  if self.input_path:
@@ -802,6 +981,9 @@ class ValueSequence(JSONLike):
802
981
 
803
982
  @property
804
983
  def workflow(self):
984
+ """
985
+ The workflow containing this sequence.
986
+ """
805
987
  if self._workflow:
806
988
  return self._workflow
807
989
  elif self._element_set:
@@ -809,6 +991,9 @@ class ValueSequence(JSONLike):
809
991
 
810
992
  @property
811
993
  def values(self):
994
+ """
995
+ The values in this sequence.
996
+ """
812
997
  if self._values_group_idx is not None:
813
998
  vals = []
814
999
  for idx, pg_idx_i in enumerate(self._values_group_idx):
@@ -885,6 +1070,9 @@ class ValueSequence(JSONLike):
885
1070
  label=None,
886
1071
  **kwargs,
887
1072
  ):
1073
+ """
1074
+ Build a sequence from a NumPy linear space.
1075
+ """
888
1076
  # TODO: save persistently as an array?
889
1077
  args = {"start": start, "stop": stop, "num": num, **kwargs}
890
1078
  values = cls._values_from_linear_space(**args)
@@ -905,6 +1093,9 @@ class ValueSequence(JSONLike):
905
1093
  label=None,
906
1094
  **kwargs,
907
1095
  ):
1096
+ """
1097
+ Build a sequence from a NumPy geometric space.
1098
+ """
908
1099
  args = {"start": start, "stop": stop, "num": num, "endpoint": endpoint, **kwargs}
909
1100
  values = cls._values_from_geometric_space(**args)
910
1101
  obj = cls(values=values, path=path, nesting_order=nesting_order, label=label)
@@ -925,6 +1116,9 @@ class ValueSequence(JSONLike):
925
1116
  label=None,
926
1117
  **kwargs,
927
1118
  ):
1119
+ """
1120
+ Build a sequence from a NumPy logarithmic space.
1121
+ """
928
1122
  args = {
929
1123
  "start": start,
930
1124
  "stop": stop,
@@ -950,6 +1144,9 @@ class ValueSequence(JSONLike):
950
1144
  label=None,
951
1145
  **kwargs,
952
1146
  ):
1147
+ """
1148
+ Build a sequence from a range.
1149
+ """
953
1150
  # TODO: save persistently as an array?
954
1151
  args = {"start": start, "stop": stop, "step": step, **kwargs}
955
1152
  if isinstance(step, int):
@@ -982,6 +1179,9 @@ class ValueSequence(JSONLike):
982
1179
  label=None,
983
1180
  **kwargs,
984
1181
  ):
1182
+ """
1183
+ Build a sequence from a simple file.
1184
+ """
985
1185
  args = {"file_path": file_path, **kwargs}
986
1186
  values = cls._values_from_file(**args)
987
1187
  obj = cls(
@@ -1009,6 +1209,8 @@ class ValueSequence(JSONLike):
1009
1209
  **kwargs,
1010
1210
  ):
1011
1211
  """
1212
+ Build a sequence to cover a rectangle.
1213
+
1012
1214
  Parameters
1013
1215
  ----------
1014
1216
  coord:
@@ -1044,6 +1246,9 @@ class ValueSequence(JSONLike):
1044
1246
  label=None,
1045
1247
  **kwargs,
1046
1248
  ):
1249
+ """
1250
+ Build a sequence from a uniform random number generator.
1251
+ """
1047
1252
  args = {"low": low, "high": high, "num": num, "seed": seed, **kwargs}
1048
1253
  values = cls._values_from_random_uniform(**args)
1049
1254
  obj = cls(values=values, path=path, nesting_order=nesting_order, label=label)
@@ -1111,6 +1316,9 @@ class AbstractInputValue(JSONLike):
1111
1316
 
1112
1317
  @property
1113
1318
  def workflow(self):
1319
+ """
1320
+ The workflow containing this input value.
1321
+ """
1114
1322
  if self._workflow:
1115
1323
  return self._workflow
1116
1324
  elif self._element_set:
@@ -1120,6 +1328,9 @@ class AbstractInputValue(JSONLike):
1120
1328
 
1121
1329
  @property
1122
1330
  def value(self):
1331
+ """
1332
+ The value itself.
1333
+ """
1123
1334
  if self._value_group_idx is not None:
1124
1335
  val = self.workflow.get_parameter_data(self._value_group_idx)
1125
1336
  if self._value_is_obj and self.parameter._value_class:
@@ -1132,31 +1343,44 @@ class AbstractInputValue(JSONLike):
1132
1343
 
1133
1344
  @dataclass
1134
1345
  class ValuePerturbation(AbstractInputValue):
1346
+ """
1347
+ A perturbation applied to a value.
1348
+ """
1349
+
1350
+ #: The name of this perturbation.
1135
1351
  name: str
1352
+ #: The path to the value(s) to perturb.
1136
1353
  path: Optional[Sequence[Union[str, int, float]]] = None
1354
+ #: The multiplicative factor to apply.
1137
1355
  multiplicative_factor: Optional[Numeric] = 1
1356
+ #: The additive factor to apply.
1138
1357
  additive_factor: Optional[Numeric] = 0
1139
1358
 
1140
1359
  @classmethod
1141
1360
  def from_spec(cls, spec):
1361
+ """
1362
+ Construct an instance from a specification dictionary.
1363
+ """
1142
1364
  return cls(**spec)
1143
1365
 
1144
1366
 
1145
1367
  class InputValue(AbstractInputValue):
1146
1368
  """
1369
+ An input value to a task.
1370
+
1147
1371
  Parameters
1148
1372
  ----------
1149
- parameter
1150
- Parameter whose value is to be specified
1151
- label
1373
+ parameter: Parameter | SchemaInput | str
1374
+ Parameter whose value is to be specified.
1375
+ label: str
1152
1376
  Optional identifier to be used where the associated `SchemaInput` accepts multiple
1153
1377
  parameters of the specified type. This will be cast to a string.
1154
- value
1378
+ value: Any
1155
1379
  The input parameter value.
1156
- value_class_method
1380
+ value_class_method: How to obtain the real value.
1157
1381
  A class method that can be invoked with the `value` attribute as keyword
1158
1382
  arguments.
1159
- path
1383
+ path: str
1160
1384
  Dot-delimited path within the parameter's nested data structure for which `value`
1161
1385
  should be set.
1162
1386
 
@@ -1188,9 +1412,16 @@ class InputValue(AbstractInputValue):
1188
1412
  elif isinstance(parameter, SchemaInput):
1189
1413
  parameter = parameter.parameter
1190
1414
 
1415
+ #: Parameter whose value is to be specified.
1191
1416
  self.parameter = parameter
1417
+ #: Identifier to be used where the associated `SchemaInput` accepts multiple
1418
+ #: parameters of the specified type.
1192
1419
  self.label = str(label) if label is not None else ""
1420
+ #: Dot-delimited path within the parameter's nested data structure for which
1421
+ #: `value` should be set.
1193
1422
  self.path = (path.strip(".") if path else None) or None
1423
+ #: A class method that can be invoked with the `value` attribute as keyword
1424
+ #: arguments.
1194
1425
  self.value_class_method = value_class_method
1195
1426
  self._value = _process_demo_data_strings(self.app, value)
1196
1427
 
@@ -1293,15 +1524,24 @@ class InputValue(AbstractInputValue):
1293
1524
 
1294
1525
  @property
1295
1526
  def labelled_type(self):
1527
+ """
1528
+ The labelled type of this input value.
1529
+ """
1296
1530
  label = f"[{self.label}]" if self.label else ""
1297
1531
  return f"{self.parameter.typ}{label}"
1298
1532
 
1299
1533
  @property
1300
1534
  def normalised_inputs_path(self):
1535
+ """
1536
+ The normalised input path without the ``inputs.`` prefix.
1537
+ """
1301
1538
  return f"{self.labelled_type}{f'.{self.path}' if self.path else ''}"
1302
1539
 
1303
1540
  @property
1304
1541
  def normalised_path(self):
1542
+ """
1543
+ The full normalised input path.
1544
+ """
1305
1545
  return f"inputs.{self.normalised_inputs_path}"
1306
1546
 
1307
1547
  def make_persistent(self, workflow: Any, source: Dict) -> Tuple[str, List[int], bool]:
@@ -1347,8 +1587,55 @@ class ResourceSpec(JSONLike):
1347
1587
  `os_name` is used for retrieving a default shell name and for retrieving the correct
1348
1588
  `Shell` class; when using WSL, it should still be `nt` (i.e. Windows).
1349
1589
 
1590
+ Parameters
1591
+ ----------
1592
+ scope:
1593
+ Which scope does this apply to.
1594
+ scratch: str
1595
+ Which scratch space to use.
1596
+ parallel_mode: ParallelMode
1597
+ Which parallel mode to use.
1598
+ num_cores: int
1599
+ How many cores to request.
1600
+ num_cores_per_node: int
1601
+ How many cores per compute node to request.
1602
+ num_threads: int
1603
+ How many threads to request.
1604
+ num_nodes: int
1605
+ How many compute nodes to request.
1606
+ scheduler: str
1607
+ Which scheduler to use.
1608
+ shell: str
1609
+ Which system shell to use.
1610
+ use_job_array: bool
1611
+ Whether to use array jobs.
1612
+ max_array_items: int
1613
+ If using array jobs, up to how many items should be in the job array.
1614
+ time_limit: str
1615
+ How long to run for.
1616
+ scheduler_args: dict[str, Any]
1617
+ Additional arguments to pass to the scheduler.
1618
+ shell_args: dict[str, Any]
1619
+ Additional arguments to pass to the shell.
1620
+ os_name: str
1621
+ Which OS to use.
1622
+ environments: dict
1623
+ Which execution environments to use.
1624
+ SGE_parallel_env: str
1625
+ Which SGE parallel environment to request.
1626
+ SLURM_partition: str
1627
+ Which SLURM partition to request.
1628
+ SLURM_num_tasks: str
1629
+ How many SLURM tasks to request.
1630
+ SLURM_num_tasks_per_node: str
1631
+ How many SLURM tasks per compute node to request.
1632
+ SLURM_num_nodes: str
1633
+ How many compute nodes to request.
1634
+ SLURM_num_cpus_per_task: str
1635
+ How many CPU cores to ask for per SLURM task.
1350
1636
  """
1351
1637
 
1638
+ #: The names of parameters that may be used when making an instance of this class.
1352
1639
  ALLOWED_PARAMETERS = {
1353
1640
  "scratch",
1354
1641
  "parallel_mode",
@@ -1407,6 +1694,7 @@ class ResourceSpec(JSONLike):
1407
1694
  SLURM_num_nodes: Optional[str] = None,
1408
1695
  SLURM_num_cpus_per_task: Optional[str] = None,
1409
1696
  ):
1697
+ #: Which scope does this apply to.
1410
1698
  self.scope = scope or self.app.ActionScope.any()
1411
1699
  if not isinstance(self.scope, self.app.ActionScope):
1412
1700
  self.scope = self.app.ActionScope.from_json_like(self.scope)
@@ -1498,10 +1786,16 @@ class ResourceSpec(JSONLike):
1498
1786
 
1499
1787
  @property
1500
1788
  def normalised_resources_path(self):
1789
+ """
1790
+ Standard name of this resource spec.
1791
+ """
1501
1792
  return self.scope.to_string()
1502
1793
 
1503
1794
  @property
1504
1795
  def normalised_path(self):
1796
+ """
1797
+ Full name of this resource spec.
1798
+ """
1505
1799
  return f"resources.{self.normalised_resources_path}"
1506
1800
 
1507
1801
  def to_dict(self):
@@ -1596,31 +1890,55 @@ class ResourceSpec(JSONLike):
1596
1890
 
1597
1891
  @property
1598
1892
  def scratch(self):
1599
- # TODO: currently unused, except in tests
1893
+ """
1894
+ Which scratch space to use.
1895
+
1896
+ Todo
1897
+ ----
1898
+ Currently unused, except in tests.
1899
+ """
1600
1900
  return self._get_value("scratch")
1601
1901
 
1602
1902
  @property
1603
1903
  def parallel_mode(self):
1904
+ """
1905
+ Which parallel mode to use.
1906
+ """
1604
1907
  return self._get_value("parallel_mode")
1605
1908
 
1606
1909
  @property
1607
1910
  def num_cores(self):
1911
+ """
1912
+ How many cores to request.
1913
+ """
1608
1914
  return self._get_value("num_cores")
1609
1915
 
1610
1916
  @property
1611
1917
  def num_cores_per_node(self):
1918
+ """
1919
+ How many cores per compute node to request.
1920
+ """
1612
1921
  return self._get_value("num_cores_per_node")
1613
1922
 
1614
1923
  @property
1615
1924
  def num_nodes(self):
1925
+ """
1926
+ How many compute nodes to request.
1927
+ """
1616
1928
  return self._get_value("num_nodes")
1617
1929
 
1618
1930
  @property
1619
1931
  def num_threads(self):
1932
+ """
1933
+ How many threads to request.
1934
+ """
1620
1935
  return self._get_value("num_threads")
1621
1936
 
1622
1937
  @property
1623
1938
  def scheduler(self):
1939
+ """
1940
+ Which scheduler to use.
1941
+ """
1624
1942
  return self._get_value("scheduler")
1625
1943
 
1626
1944
  @scheduler.setter
@@ -1631,6 +1949,9 @@ class ResourceSpec(JSONLike):
1631
1949
 
1632
1950
  @property
1633
1951
  def shell(self):
1952
+ """
1953
+ Which system shell to use.
1954
+ """
1634
1955
  return self._get_value("shell")
1635
1956
 
1636
1957
  @shell.setter
@@ -1641,54 +1962,93 @@ class ResourceSpec(JSONLike):
1641
1962
 
1642
1963
  @property
1643
1964
  def use_job_array(self):
1965
+ """
1966
+ Whether to use array jobs.
1967
+ """
1644
1968
  return self._get_value("use_job_array")
1645
1969
 
1646
1970
  @property
1647
1971
  def max_array_items(self):
1972
+ """
1973
+ If using array jobs, up to how many items should be in the job array.
1974
+ """
1648
1975
  return self._get_value("max_array_items")
1649
1976
 
1650
1977
  @property
1651
1978
  def time_limit(self):
1979
+ """
1980
+ How long to run for.
1981
+ """
1652
1982
  return self._get_value("time_limit")
1653
1983
 
1654
1984
  @property
1655
1985
  def scheduler_args(self):
1986
+ """
1987
+ Additional arguments to pass to the scheduler.
1988
+ """
1656
1989
  return self._get_value("scheduler_args")
1657
1990
 
1658
1991
  @property
1659
1992
  def shell_args(self):
1993
+ """
1994
+ Additional arguments to pass to the shell.
1995
+ """
1660
1996
  return self._get_value("shell_args")
1661
1997
 
1662
1998
  @property
1663
1999
  def os_name(self):
2000
+ """
2001
+ Which OS to use.
2002
+ """
1664
2003
  return self._get_value("os_name")
1665
2004
 
1666
2005
  @property
1667
2006
  def environments(self):
2007
+ """
2008
+ Which execution environments to use.
2009
+ """
1668
2010
  return self._get_value("environments")
1669
2011
 
1670
2012
  @property
1671
2013
  def SGE_parallel_env(self):
2014
+ """
2015
+ Which SGE parallel environment to request.
2016
+ """
1672
2017
  return self._get_value("SGE_parallel_env")
1673
2018
 
1674
2019
  @property
1675
2020
  def SLURM_partition(self):
2021
+ """
2022
+ Which SLURM partition to request.
2023
+ """
1676
2024
  return self._get_value("SLURM_partition")
1677
2025
 
1678
2026
  @property
1679
2027
  def SLURM_num_tasks(self):
2028
+ """
2029
+ How many SLURM tasks to request.
2030
+ """
1680
2031
  return self._get_value("SLURM_num_tasks")
1681
2032
 
1682
2033
  @property
1683
2034
  def SLURM_num_tasks_per_node(self):
2035
+ """
2036
+ How many SLURM tasks per compute node to request.
2037
+ """
1684
2038
  return self._get_value("SLURM_num_tasks_per_node")
1685
2039
 
1686
2040
  @property
1687
2041
  def SLURM_num_nodes(self):
2042
+ """
2043
+ How many compute nodes to request.
2044
+ """
1688
2045
  return self._get_value("SLURM_num_nodes")
1689
2046
 
1690
2047
  @property
1691
2048
  def SLURM_num_cpus_per_task(self):
2049
+ """
2050
+ How many CPU cores to ask for per SLURM task.
2051
+ """
1692
2052
  return self._get_value("SLURM_num_cpus_per_task")
1693
2053
 
1694
2054
  @os_name.setter
@@ -1698,6 +2058,9 @@ class ResourceSpec(JSONLike):
1698
2058
 
1699
2059
  @property
1700
2060
  def workflow(self):
2061
+ """
2062
+ The workflow owning this resource spec.
2063
+ """
1701
2064
  if self._workflow:
1702
2065
  return self._workflow
1703
2066
 
@@ -1718,27 +2081,69 @@ class ResourceSpec(JSONLike):
1718
2081
 
1719
2082
  @property
1720
2083
  def element_set(self):
2084
+ """
2085
+ The element set that will use this resource spec.
2086
+ """
1721
2087
  return self._resource_list.element_set
1722
2088
 
1723
2089
  @property
1724
2090
  def workflow_template(self):
2091
+ """
2092
+ The workflow template that will use this resource spec.
2093
+ """
1725
2094
  return self._resource_list.workflow_template
1726
2095
 
1727
2096
 
1728
2097
  class InputSourceType(enum.Enum):
2098
+ """
2099
+ The types if input sources.
2100
+ """
2101
+
2102
+ #: Input source is an import.
1729
2103
  IMPORT = 0
2104
+ #: Input source is local.
1730
2105
  LOCAL = 1
2106
+ #: Input source is a default.
1731
2107
  DEFAULT = 2
2108
+ #: Input source is a task.
1732
2109
  TASK = 3
1733
2110
 
1734
2111
 
1735
2112
  class TaskSourceType(enum.Enum):
2113
+ """
2114
+ The types of task-based input sources.
2115
+ """
2116
+
2117
+ #: Input source is a task input.
1736
2118
  INPUT = 0
2119
+ #: Input source is a task output.
1737
2120
  OUTPUT = 1
2121
+ #: Input source is unspecified.
1738
2122
  ANY = 2
1739
2123
 
1740
2124
 
1741
2125
  class InputSource(JSONLike):
2126
+ """
2127
+ An input source to a workflow task.
2128
+
2129
+ Parameters
2130
+ ----------
2131
+ source_type: InputSourceType
2132
+ Type of the input source.
2133
+ import_ref:
2134
+ Where the input comes from when the type is `IMPORT`.
2135
+ task_ref:
2136
+ Which task is this an input for? Used when the type is `TASK`.
2137
+ task_source_type: TaskSourceType
2138
+ Type of task source.
2139
+ element_iters:
2140
+ Which element iterations does this apply to?
2141
+ path:
2142
+ Path to where this input goes.
2143
+ where: ~hpcflow.app.Rule | list[~hpcflow.app.Rule] | ~hpcflow.app.ElementFilter
2144
+ Filtering rules.
2145
+ """
2146
+
1742
2147
  _child_objects = (
1743
2148
  ChildObjectSpec(
1744
2149
  name="source_type",
@@ -1769,12 +2174,19 @@ class InputSource(JSONLike):
1769
2174
  rules[idx] = app.Rule(**i)
1770
2175
  where = app.ElementFilter(rules=rules)
1771
2176
 
2177
+ #: Type of the input source.
1772
2178
  self.source_type = self._validate_source_type(source_type)
2179
+ #: Where the input comes from when the type is `IMPORT`.
1773
2180
  self.import_ref = import_ref
2181
+ #: Which task is this an input for? Used when the type is `TASK`.
1774
2182
  self.task_ref = task_ref
2183
+ #: Type of task source.
1775
2184
  self.task_source_type = self._validate_task_source_type(task_source_type)
2185
+ #: Which element iterations does this apply to?
1776
2186
  self.element_iters = element_iters
2187
+ #: Filtering rules.
1777
2188
  self.where = where
2189
+ #: Path to where this input goes.
1778
2190
  self.path = path
1779
2191
 
1780
2192
  if self.source_type is InputSourceType.TASK:
@@ -1852,6 +2264,9 @@ class InputSource(JSONLike):
1852
2264
  return None
1853
2265
 
1854
2266
  def to_string(self):
2267
+ """
2268
+ Render this input source as a string.
2269
+ """
1855
2270
  out = [self.source_type.name.lower()]
1856
2271
  if self.source_type is InputSourceType.TASK:
1857
2272
  out += [str(self.task_ref), self.task_source_type.name.lower()]
@@ -1893,20 +2308,26 @@ class InputSource(JSONLike):
1893
2308
 
1894
2309
  @classmethod
1895
2310
  def from_string(cls, str_defn):
1896
- return cls(**cls._parse_from_string(str_defn))
1897
-
1898
- @classmethod
1899
- def _parse_from_string(cls, str_defn):
1900
2311
  """Parse a dot-delimited string definition of an InputSource.
1901
2312
 
1902
- Examples:
1903
- - task.[task_ref].input
1904
- - task.[task_ref].output
1905
- - local
1906
- - default
1907
- - import.[import_ref]
2313
+ Parameter
2314
+ ---------
2315
+ str_defn:
2316
+ The string to parse.
2317
+
2318
+ Examples
2319
+ --------
2320
+ task.[task_ref].input
2321
+ task.[task_ref].output
2322
+ local
2323
+ default
2324
+ import.[import_ref]
1908
2325
 
1909
2326
  """
2327
+ return cls(**cls._parse_from_string(str_defn))
2328
+
2329
+ @classmethod
2330
+ def _parse_from_string(cls, str_defn):
1910
2331
  parts = str_defn.split(".")
1911
2332
  source_type = cls._validate_source_type(parts[0])
1912
2333
  task_ref = None
@@ -1957,6 +2378,18 @@ class InputSource(JSONLike):
1957
2378
 
1958
2379
  @classmethod
1959
2380
  def import_(cls, import_ref, element_iters=None, where=None):
2381
+ """
2382
+ Make an instnace of an input source that is an import.
2383
+
2384
+ Parameters
2385
+ ----------
2386
+ import_ref:
2387
+ Import reference.
2388
+ element_iters:
2389
+ Originating element iterations.
2390
+ where:
2391
+ Filtering rule.
2392
+ """
1960
2393
  return cls(
1961
2394
  source_type=cls.app.InputSourceType.IMPORT,
1962
2395
  import_ref=import_ref,
@@ -1966,14 +2399,34 @@ class InputSource(JSONLike):
1966
2399
 
1967
2400
  @classmethod
1968
2401
  def local(cls):
2402
+ """
2403
+ Make an instnace of an input source that is local.
2404
+ """
1969
2405
  return cls(source_type=cls.app.InputSourceType.LOCAL)
1970
2406
 
1971
2407
  @classmethod
1972
2408
  def default(cls):
2409
+ """
2410
+ Make an instnace of an input source that is default.
2411
+ """
1973
2412
  return cls(source_type=cls.app.InputSourceType.DEFAULT)
1974
2413
 
1975
2414
  @classmethod
1976
2415
  def task(cls, task_ref, task_source_type=None, element_iters=None, where=None):
2416
+ """
2417
+ Make an instnace of an input source that is a task.
2418
+
2419
+ Parameters
2420
+ ----------
2421
+ task_ref:
2422
+ Source task reference.
2423
+ task_source_type:
2424
+ Type of task source.
2425
+ element_iters:
2426
+ Originating element iterations.
2427
+ where:
2428
+ Filtering rule.
2429
+ """
1977
2430
  if not task_source_type:
1978
2431
  task_source_type = cls.app.TaskSourceType.OUTPUT
1979
2432
  return cls(