hpcflow-new2 0.2.0a69__py3-none-any.whl → 0.2.0a71__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,7 +15,7 @@ from hpcflow.sdk.core.errors import (
15
15
  WorkflowParameterMissingError,
16
16
  )
17
17
  from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
18
- from hpcflow.sdk.core.utils import check_valid_py_identifier
18
+ from hpcflow.sdk.core.utils import check_valid_py_identifier, get_enum_by_name_or_val
19
19
  from hpcflow.sdk.submission.submission import timedelta_format
20
20
 
21
21
 
@@ -148,10 +148,47 @@ class SchemaParameter(JSONLike):
148
148
  return self.parameter.typ
149
149
 
150
150
 
151
- @dataclass
151
+ class NullDefault(enum.Enum):
152
+ NULL = 0
153
+
154
+
152
155
  class SchemaInput(SchemaParameter):
153
156
  """A Parameter as used within a particular schema, for which a default value may be
154
- applied."""
157
+ applied.
158
+
159
+ Parameters
160
+ ----------
161
+ parameter
162
+ The parameter (i.e. type) of this schema input.
163
+ multiple
164
+ If True, expect one or more of these parameters defined in the workflow,
165
+ distinguished by a string label in square brackets. For example `p1[0]` for a
166
+ parameter `p1`.
167
+ labels
168
+ Dict whose keys represent the string labels that distinguish multiple parameters
169
+ if `multiple` is `True`. Use the key "*" to mean all labels not matching
170
+ other label keys. If `multiple` is `False`, this will default to a
171
+ single-item dict with an empty string key: `{{"": {{}}}}`. If `multiple` is
172
+ `True`, this will default to a single-item dict with the catch-all key:
173
+ `{{"*": {{}}}}`. On initialisation, remaining keyword-arguments are treated as default
174
+ values for the dict values of `labels`.
175
+ default_value
176
+ The default value for this input parameter. This is itself a default value that
177
+ will be applied to all `labels` values if a "default_value" key does not exist.
178
+ propagation_mode
179
+ Determines how this input should propagate through the workflow. This is a default
180
+ value that will be applied to all `labels` values if a "propagation_mode" key does
181
+ not exist. By default, the input is allowed to be used in downstream tasks simply
182
+ because it has a compatible type (this is the "implicit" propagation mode). Other
183
+ options are "explicit", meaning that the parameter must be explicitly specified in
184
+ the downstream task `input_sources` for it to be used, and "never", meaning that
185
+ the parameter must not be used in downstream tasks and will be inaccessible to
186
+ those tasks.
187
+ group
188
+ Determines the name of the element group from which this input should be sourced.
189
+ This is a default value that will be applied to all `labels` if a "group" key
190
+ does not exist.
191
+ """
155
192
 
156
193
  _task_schema = None # assigned by parent TaskSchema
157
194
 
@@ -162,74 +199,142 @@ class SchemaInput(SchemaParameter):
162
199
  shared_data_name="parameters",
163
200
  shared_data_primary_key="typ",
164
201
  ),
165
- ChildObjectSpec(
166
- name="default_value",
167
- class_name="InputValue",
168
- parent_ref="_schema_input",
169
- ),
170
- ChildObjectSpec(
171
- name="propagation_mode",
172
- class_name="ParameterPropagationMode",
173
- is_enum=True,
174
- ),
175
202
  )
176
203
 
177
- parameter: app.Parameter
178
- default_value: Optional[app.InputValue] = None
179
- propagation_mode: ParameterPropagationMode = ParameterPropagationMode.IMPLICIT
204
+ def __init__(
205
+ self,
206
+ parameter: app.Parameter,
207
+ multiple: bool = False,
208
+ labels: Optional[Dict] = None,
209
+ default_value: Optional[Union[app.InputValue, NullDefault]] = NullDefault.NULL,
210
+ propagation_mode: ParameterPropagationMode = ParameterPropagationMode.IMPLICIT,
211
+ group: Optional[str] = None,
212
+ ):
213
+ # TODO: can we define elements groups on local inputs as well, or should these be
214
+ # just for elements from other tasks?
180
215
 
181
- # can we define elements groups on local inputs as well, or should these be just for
182
- # elements from other tasks?
183
- group: Optional[str] = None
184
- where: Optional[app.ElementFilter] = None
216
+ # TODO: test we allow unlabelled with accepts-multiple True.
217
+ # TODO: test we allow a single labelled with accepts-multiple False.
218
+
219
+ if not isinstance(parameter, app.Parameter):
220
+ parameter = app.Parameter(parameter)
221
+
222
+ self.parameter = parameter
223
+ self.multiple = multiple
224
+ self.labels = labels
225
+
226
+ if self.labels is None:
227
+ if self.multiple:
228
+ self.labels = {"*": {}}
229
+ else:
230
+ self.labels = {"": {}}
231
+ else:
232
+ if not self.multiple:
233
+ # check single-item:
234
+ if len(self.labels) > 1:
235
+ raise ValueError(
236
+ f"If `{self.__class__.__name__}.multiple` is `False`, "
237
+ f"then `labels` must be a single-item `dict` if specified, but "
238
+ f"`labels` is: {self.labels!r}."
239
+ )
240
+
241
+ labels_defaults = {}
242
+ if propagation_mode is not None:
243
+ labels_defaults["propagation_mode"] = propagation_mode
244
+ if group is not None:
245
+ labels_defaults["group"] = group
246
+
247
+ # apply defaults:
248
+ for k, v in self.labels.items():
249
+ labels_defaults_i = copy.deepcopy(labels_defaults)
250
+ if default_value is not NullDefault.NULL:
251
+ if not isinstance(default_value, InputValue):
252
+ default_value = app.InputValue(
253
+ parameter=self.parameter,
254
+ value=default_value,
255
+ label=k,
256
+ )
257
+ labels_defaults_i["default_value"] = default_value
258
+ label_i = {**labels_defaults_i, **v}
259
+ if "propagation_mode" in label_i:
260
+ label_i["propagation_mode"] = get_enum_by_name_or_val(
261
+ ParameterPropagationMode, label_i["propagation_mode"]
262
+ )
263
+ if "default_value" in label_i:
264
+ label_i["default_value"]._schema_input = self
265
+ self.labels[k] = label_i
185
266
 
186
- def __post_init__(self):
187
- if not isinstance(self.propagation_mode, ParameterPropagationMode):
188
- self.propagation_mode = getattr(
189
- ParameterPropagationMode, self.propagation_mode.upper()
190
- )
191
- super().__post_init__()
192
267
  self._set_parent_refs()
268
+ self._validate()
193
269
 
194
270
  def __repr__(self) -> str:
195
271
  default_str = ""
196
- if self.default_value is not None:
197
- default_str = f", default_value={self.default_value!r}"
198
-
199
272
  group_str = ""
200
- if self.group is not None:
201
- group_str = f", group={self.group!r}"
273
+ labels_str = ""
274
+ if not self.multiple:
275
+ label = next(iter(self.labels.keys())) # the single key
276
+
277
+ default_str = ""
278
+ if "default_value" in self.labels[label]:
279
+ default_str = (
280
+ f", default_value={self.labels[label]['default_value'].value!r}"
281
+ )
202
282
 
203
- where_str = ""
204
- if self.where is not None:
205
- where_str = f", group={self.where!r}"
283
+ group = self.labels[label].get("group")
284
+ if group is not None:
285
+ group_str = f", group={group!r}"
286
+
287
+ else:
288
+ labels_str = f", labels={str(self.labels)!r}"
206
289
 
207
290
  return (
208
291
  f"{self.__class__.__name__}("
209
292
  f"parameter={self.parameter.__class__.__name__}({self.parameter.typ!r}), "
210
- f"propagation_mode={self.propagation_mode.name!r}"
211
- f"{default_str}{group_str}{where_str}"
293
+ f"multiple={self.multiple!r}"
294
+ f"{default_str}{group_str}{labels_str}"
212
295
  f")"
213
296
  )
214
297
 
298
+ def to_dict(self):
299
+ dct = super().to_dict()
300
+ for k, v in dct["labels"].items():
301
+ prop_mode = v.get("parameter_propagation_mode")
302
+ if prop_mode:
303
+ dct["labels"][k]["parameter_propagation_mode"] = prop_mode.name
304
+ return dct
305
+
306
+ def to_json_like(self, dct=None, shared_data=None, exclude=None, path=None):
307
+ out, shared = super().to_json_like(dct, shared_data, exclude, path)
308
+ for k, v in out["labels"].items():
309
+ if "default_value" in v:
310
+ out["labels"][k]["default_value_is_input_value"] = True
311
+ return out, shared
312
+
215
313
  @classmethod
216
314
  def from_json_like(cls, json_like, shared_data=None):
217
- # we assume if default_value is specified, it is only the `value` part of the
218
- # `InputValue` JSON:
219
- if "default_value" in json_like:
220
- json_like["default_value"] = {
221
- "parameter": json_like["parameter"],
222
- "value": json_like["default_value"],
223
- }
315
+ for k, v in json_like.get("labels", {}).items():
316
+ if "default_value" in v:
317
+ if "default_value_is_input_value" in v:
318
+ inp_val_kwargs = v["default_value"]
319
+ else:
320
+ inp_val_kwargs = {
321
+ "parameter": json_like["parameter"],
322
+ "value": v["default_value"],
323
+ "label": k,
324
+ }
325
+ json_like["labels"][k]["default_value"] = InputValue.from_json_like(
326
+ json_like=inp_val_kwargs,
327
+ shared_data=shared_data,
328
+ )
329
+
224
330
  obj = super().from_json_like(json_like, shared_data)
225
331
  return obj
226
332
 
227
333
  def __deepcopy__(self, memo):
228
334
  kwargs = {
229
335
  "parameter": self.parameter,
230
- "default_value": self.default_value,
231
- "propagation_mode": self.propagation_mode,
232
- "group": self.group,
336
+ "multiple": self.multiple,
337
+ "labels": self.labels,
233
338
  }
234
339
  obj = self.__class__(**copy.deepcopy(kwargs, memo))
235
340
  obj._task_schema = self._task_schema
@@ -239,20 +344,51 @@ class SchemaInput(SchemaParameter):
239
344
  def task_schema(self):
240
345
  return self._task_schema
241
346
 
347
+ @property
348
+ def all_labelled_types(self):
349
+ return list(f"{self.typ}{f'[{i}]' if i else ''}" for i in self.labels)
350
+
351
+ @property
352
+ def single_label(self):
353
+ if not self.multiple:
354
+ return next(iter(self.labels))
355
+
356
+ @property
357
+ def single_labelled_type(self):
358
+ if not self.multiple:
359
+ return next(iter(self.labelled_info()))["labelled_type"]
360
+
361
+ def labelled_info(self):
362
+ for k, v in self.labels.items():
363
+ label = f"[{k}]" if k else ""
364
+ dct = {
365
+ "labelled_type": self.parameter.typ + label,
366
+ "propagation_mode": v["propagation_mode"],
367
+ "group": v.get("group"),
368
+ }
369
+ if "default_value" in v:
370
+ dct["default_value"] = v["default_value"]
371
+ yield dct
372
+
242
373
  def _validate(self):
243
374
  super()._validate()
244
- if self.default_value is not None:
245
- if not isinstance(self.default_value, InputValue):
246
- self.default_value = self.app.InputValue(
247
- parameter=self.parameter,
248
- value=self.default_value,
249
- )
250
- if self.default_value.parameter != self.parameter:
251
- raise ValueError(
252
- f"{self.__class__.__name__} `default_value` must be an `InputValue` for "
253
- f"parameter: {self.parameter!r}, but specified `InputValue` parameter "
254
- f"is: {self.default_value.parameter!r}."
255
- )
375
+ for k, v in self.labels.items():
376
+ if "default_value" in v:
377
+ if not isinstance(v["default_value"], InputValue):
378
+ def_val = self.app.InputValue(
379
+ parameter=self.parameter,
380
+ value=v["default_value"],
381
+ label=k,
382
+ )
383
+ self.labels[k]["default_value"] = def_val
384
+ def_val = self.labels[k]["default_value"]
385
+ if def_val.parameter != self.parameter or def_val.label != k:
386
+ raise ValueError(
387
+ f"{self.__class__.__name__} `default_value` for label {k!r} must "
388
+ f"be an `InputValue` for parameter: {self.parameter!r} with the "
389
+ f"same label, but specified `InputValue` is: "
390
+ f"{v['default_value']!r}."
391
+ )
256
392
 
257
393
  @property
258
394
  def input_or_output(self):
@@ -296,9 +432,14 @@ class ValueSequence(JSONLike):
296
432
  path: str,
297
433
  nesting_order: int,
298
434
  values: List[Any],
435
+ label: Optional[str] = None,
299
436
  value_class_method: Optional[str] = None,
300
437
  ):
301
- self.path = self._validate_parameter_path(path)
438
+ label = str(label) if label is not None else ""
439
+ path, label = self._validate_parameter_path(path, label)
440
+
441
+ self.path = path
442
+ self.label = label
302
443
  self.nesting_order = nesting_order
303
444
  self.value_class_method = value_class_method
304
445
 
@@ -319,6 +460,9 @@ class ValueSequence(JSONLike):
319
460
  self._values_method_args = None
320
461
 
321
462
  def __repr__(self):
463
+ label_str = ""
464
+ if self.label:
465
+ label_str = f"label={self.label!r}, "
322
466
  vals_grp_idx = (
323
467
  f"values_group_idx={self._values_group_idx}, "
324
468
  if self._values_group_idx
@@ -327,6 +471,7 @@ class ValueSequence(JSONLike):
327
471
  return (
328
472
  f"{self.__class__.__name__}("
329
473
  f"path={self.path!r}, "
474
+ f"{label_str}"
330
475
  f"nesting_order={self.nesting_order}, "
331
476
  f"{vals_grp_idx}"
332
477
  f"values={self.values}"
@@ -404,12 +549,12 @@ class ValueSequence(JSONLike):
404
549
  @property
405
550
  def input_type(self):
406
551
  if self.path_type == "inputs":
407
- return self.path_split[1]
552
+ return self.path_split[1].replace(self._label_fmt, "")
408
553
 
409
554
  @property
410
555
  def input_path(self):
411
556
  if self.path_type == "inputs":
412
- return ".".join(self.path_split[2:]) or None
557
+ return ".".join(self.path_split[2:])
413
558
 
414
559
  @property
415
560
  def resource_scope(self):
@@ -421,6 +566,15 @@ class ValueSequence(JSONLike):
421
566
  """True if the values are for a sub part of the parameter."""
422
567
  return True if self.input_path else False
423
568
 
569
+ @property
570
+ def _label_fmt(self):
571
+ return f"[{self.label}]" if self.label else ""
572
+
573
+ @property
574
+ def labelled_type(self):
575
+ if self.input_type:
576
+ return f"{self.input_type}{self._label_fmt}"
577
+
424
578
  @classmethod
425
579
  def _json_like_constructor(cls, json_like):
426
580
  """Invoked by `JSONLike.from_json_like` instead of `__init__`."""
@@ -439,7 +593,15 @@ class ValueSequence(JSONLike):
439
593
  obj._values_method_args = _values_method_args
440
594
  return obj
441
595
 
442
- def _validate_parameter_path(self, path):
596
+ def _validate_parameter_path(self, path, label):
597
+ """Parse the supplied path and perform basic checks on it.
598
+
599
+ This method also adds the specified `SchemaInput` label to the path and checks for
600
+ consistency if a label is already present.
601
+
602
+ """
603
+ label_arg = label
604
+
443
605
  if not isinstance(path, str):
444
606
  raise MalformedParameterPathError(
445
607
  f"`path` must be a string, but given path has type {type(path)} with value "
@@ -451,7 +613,36 @@ class ValueSequence(JSONLike):
451
613
  f'`path` must start with "inputs", "outputs", or "resources", but given path '
452
614
  f"is: {path!r}."
453
615
  )
616
+
617
+ try:
618
+ label_from_path = path_split[1].split("[")[1].split("]")[0]
619
+ except IndexError:
620
+ label_from_path = None
621
+
622
+ if path_split[0] == "inputs":
623
+ if label_arg:
624
+ if not label_from_path:
625
+ # add label to path without lower casing any parts:
626
+ path_split_orig = path.split(".")
627
+ path_split_orig[1] += f"[{label_arg}]"
628
+ path = ".".join(path_split_orig)
629
+ label = label_arg
630
+ elif label_arg != label_from_path:
631
+ raise ValueError(
632
+ f"{self.__class__.__name__} `label` argument is specified as "
633
+ f"{label_arg!r}, but a distinct label is implied by the sequence "
634
+ f"path: {path!r}."
635
+ )
636
+ elif label_from_path:
637
+ label = label_from_path
638
+
454
639
  if path_split[0] == "resources":
640
+ if label_from_path or label_arg:
641
+ raise ValueError(
642
+ f"{self.__class__.__name__} `label` argument ({label_arg!r}) and/or "
643
+ f"label specification via `path` ({path!r}) is not supported for "
644
+ f"`resource` sequences."
645
+ )
455
646
  try:
456
647
  self.app.ActionScope.from_json_like(path_split[1])
457
648
  except Exception as err:
@@ -470,7 +661,7 @@ class ValueSequence(JSONLike):
470
661
  f"resource item names are: {allowed_keys_str}."
471
662
  )
472
663
 
473
- return path
664
+ return path, label
474
665
 
475
666
  def to_dict(self):
476
667
  out = super().to_dict()
@@ -490,7 +681,10 @@ class ValueSequence(JSONLike):
490
681
  inputs sequence, else return None."""
491
682
 
492
683
  if self.input_type:
493
- return ".".join(self.path_split[1:])
684
+ if self.input_path:
685
+ return f"{self.labelled_type}.{self.input_path}"
686
+ else:
687
+ return self.labelled_type
494
688
 
495
689
  def make_persistent(
496
690
  self, workflow: app.Workflow, source: Dict
@@ -541,14 +735,28 @@ class ValueSequence(JSONLike):
541
735
  if self._values_group_idx is not None:
542
736
  vals = []
543
737
  for idx, pg_idx_i in enumerate(self._values_group_idx):
544
- val = self.workflow.get_parameter_data(pg_idx_i)
738
+ param_i = self.workflow.get_parameter(pg_idx_i)
739
+ if param_i.data is not None:
740
+ val_i = param_i.data
741
+ else:
742
+ val_i = param_i.file
743
+
744
+ # `val_i` might already be a `_value_class` object if the store has not
745
+ # yet been committed to disk:
545
746
  if (
546
747
  self.parameter
547
748
  and self.parameter._value_class
548
749
  and self._values_are_objs[idx]
750
+ and not isinstance(val_i, self.parameter._value_class)
549
751
  ):
550
- val = self.parameter._value_class(**val)
551
- vals.append(val)
752
+ method_name = param_i.source.get("value_class_method")
753
+ if method_name:
754
+ method = getattr(self.parameter._value_class, method_name)
755
+ else:
756
+ method = self.parameter._value_class
757
+ val_i = method(**val_i)
758
+
759
+ vals.append(val_i)
552
760
  return vals
553
761
  else:
554
762
  return self._values
@@ -619,6 +827,8 @@ class AbstractInputValue(JSONLike):
619
827
  out = super().to_dict()
620
828
  if "_workflow" in out:
621
829
  del out["_workflow"]
830
+ if "_schema_input" in out:
831
+ del out["_schema_input"]
622
832
  return out
623
833
 
624
834
  def make_persistent(
@@ -685,6 +895,25 @@ class ValuePerturbation(AbstractInputValue):
685
895
 
686
896
 
687
897
  class InputValue(AbstractInputValue):
898
+ """
899
+ Parameters
900
+ ----------
901
+ parameter
902
+ Parameter whose value is to be specified
903
+ label
904
+ Optional identifier to be used where the associated `SchemaInput` accepts multiple
905
+ parameters of the specified type. This will be cast to a string.
906
+ value
907
+ The input parameter value.
908
+ value_class_method
909
+ A class method that can be invoked with the `value` attribute as keyword
910
+ arguments.
911
+ path
912
+ Dot-delimited path within the parameter's nested data structure for which `value`
913
+ should be set.
914
+
915
+ """
916
+
688
917
  _child_objects = (
689
918
  ChildObjectSpec(
690
919
  name="parameter",
@@ -698,6 +927,7 @@ class InputValue(AbstractInputValue):
698
927
  self,
699
928
  parameter: Union[app.Parameter, str],
700
929
  value: Optional[Any] = None,
930
+ label: Optional[str] = None,
701
931
  value_class_method: Optional[str] = None,
702
932
  path: Optional[str] = None,
703
933
  ):
@@ -707,6 +937,7 @@ class InputValue(AbstractInputValue):
707
937
  parameter = parameter.parameter
708
938
 
709
939
  self.parameter = parameter
940
+ self.label = str(label) if label is not None else ""
710
941
  self.path = (path.strip(".") if path else None) or None
711
942
  self.value_class_method = value_class_method
712
943
  self._value = value
@@ -725,6 +956,7 @@ class InputValue(AbstractInputValue):
725
956
  def __deepcopy__(self, memo):
726
957
  kwargs = self.to_dict()
727
958
  _value = kwargs.pop("_value")
959
+ kwargs.pop("_schema_input", None)
728
960
  _value_group_idx = kwargs.pop("_value_group_idx")
729
961
  _value_is_obj = kwargs.pop("_value_is_obj")
730
962
  obj = self.__class__(**copy.deepcopy(kwargs, memo))
@@ -744,6 +976,10 @@ class InputValue(AbstractInputValue):
744
976
  if self.path is not None:
745
977
  path_str = f", path={self.path!r}"
746
978
 
979
+ label_str = ""
980
+ if self.label is not None:
981
+ label_str = f", label={self.label!r}"
982
+
747
983
  try:
748
984
  value_str = f", value={self.value}"
749
985
  except WorkflowParameterMissingError:
@@ -751,7 +987,7 @@ class InputValue(AbstractInputValue):
751
987
 
752
988
  return (
753
989
  f"{self.__class__.__name__}("
754
- f"parameter={self.parameter.typ!r}"
990
+ f"parameter={self.parameter.typ!r}{label_str}"
755
991
  f"{value_str}"
756
992
  f"{path_str}"
757
993
  f"{val_grp_idx}"
@@ -780,9 +1016,14 @@ class InputValue(AbstractInputValue):
780
1016
 
781
1017
  return obj
782
1018
 
1019
+ @property
1020
+ def labelled_type(self):
1021
+ label = f"[{self.label}]" if self.label else ""
1022
+ return f"{self.parameter.typ}{label}"
1023
+
783
1024
  @property
784
1025
  def normalised_inputs_path(self):
785
- return f"{self.parameter.typ}{f'.{self.path}' if self.path else ''}"
1026
+ return f"{self.labelled_type}{f'.{self.path}' if self.path else ''}"
786
1027
 
787
1028
  @property
788
1029
  def normalised_path(self):
@@ -795,6 +1036,12 @@ class InputValue(AbstractInputValue):
795
1036
 
796
1037
  @classmethod
797
1038
  def from_json_like(cls, json_like, shared_data=None):
1039
+ if "[" in json_like["parameter"]:
1040
+ # extract out the parameter label:
1041
+ label = json_like["parameter"].split("[")[1].split("]")[0]
1042
+ json_like["parameter"] = json_like["parameter"].replace(f"[{label}]", "")
1043
+ json_like["label"] = label
1044
+
798
1045
  if "::" in json_like["parameter"]:
799
1046
  param, cls_method = json_like["parameter"].split("::")
800
1047
  json_like["parameter"] = param