hpcflow-new2 0.2.0a69__py3-none-any.whl → 0.2.0a71__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
hpcflow/sdk/core/task.py CHANGED
@@ -113,7 +113,7 @@ class ElementSet(JSONLike):
113
113
  input_files: Optional[List[app.InputFile]] = None,
114
114
  sequences: Optional[List[app.ValueSequence]] = None,
115
115
  resources: Optional[Dict[str, Dict]] = None,
116
- repeats: Optional[Union[int, List[int]]] = 1,
116
+ repeats: Optional[List[Dict]] = None,
117
117
  groups: Optional[List[app.ElementGroup]] = None,
118
118
  input_sources: Optional[Dict[str, app.InputSource]] = None,
119
119
  nesting_order: Optional[List] = None,
@@ -143,7 +143,7 @@ class ElementSet(JSONLike):
143
143
 
144
144
  self.inputs = inputs or []
145
145
  self.input_files = input_files or []
146
- self.repeats = repeats
146
+ self.repeats = repeats or []
147
147
  self.groups = groups or []
148
148
  self.resources = resources
149
149
  self.sequences = sequences or []
@@ -216,7 +216,7 @@ class ElementSet(JSONLike):
216
216
 
217
217
  @property
218
218
  def input_types(self):
219
- return [i.parameter.typ for i in self.inputs]
219
+ return [i.labelled_type for i in self.inputs]
220
220
 
221
221
  @property
222
222
  def element_local_idx_range(self):
@@ -266,7 +266,7 @@ class ElementSet(JSONLike):
266
266
 
267
267
  seq_inp_types = []
268
268
  for seq_i in self.sequences:
269
- inp_type = seq_i.input_type
269
+ inp_type = seq_i.labelled_type
270
270
  if inp_type:
271
271
  bad_inp = {inp_type} - self.task_template.all_schema_input_types
272
272
  allowed_str = ", ".join(
@@ -282,6 +282,11 @@ class ElementSet(JSONLike):
282
282
  if seq_i.path not in self.nesting_order:
283
283
  self.nesting_order.update({seq_i.path: seq_i.nesting_order})
284
284
 
285
+ for rep_spec in self.repeats:
286
+ reps_path_i = f'repeats.{rep_spec["name"]}'
287
+ if reps_path_i not in self.nesting_order:
288
+ self.nesting_order[reps_path_i] = rep_spec["nesting_order"]
289
+
285
290
  for k, v in self.nesting_order.items():
286
291
  if v < 0:
287
292
  raise TaskTemplateInvalidNesting(
@@ -408,22 +413,44 @@ class ElementSet(JSONLike):
408
413
 
409
414
  return deps
410
415
 
411
- def is_input_type_provided(self, typ: str) -> bool:
416
+ def is_input_type_provided(self, labelled_path: str) -> bool:
412
417
  """Check if an input is provided locally as an InputValue or a ValueSequence."""
413
418
 
414
419
  for inp in self.inputs:
415
- if typ == inp.parameter.typ:
420
+ if labelled_path == inp.normalised_inputs_path:
416
421
  return True
417
422
 
418
423
  for seq in self.sequences:
419
424
  if seq.parameter:
420
425
  # i.e. not a resource:
421
- if not seq.is_sub_value and typ == seq.parameter.typ:
426
+ if labelled_path == seq.normalised_inputs_path:
422
427
  return True
423
428
 
424
429
  return False
425
430
 
426
431
 
432
+ class OutputLabel(JSONLike):
433
+ """Class to represent schema input labels that should be applied to a subset of task
434
+ outputs"""
435
+
436
+ _child_objects = (
437
+ ChildObjectSpec(
438
+ name="where",
439
+ class_name="ElementFilter",
440
+ ),
441
+ )
442
+
443
+ def __init__(
444
+ self,
445
+ parameter: str,
446
+ label: str,
447
+ where: Optional[List[app.ElementFilter]] = None,
448
+ ) -> None:
449
+ self.parameter = parameter
450
+ self.label = label
451
+ self.where = where
452
+
453
+
427
454
  class Task(JSONLike):
428
455
  """Parametrisation of an isolated task for which a subset of input values are given
429
456
  "locally". The remaining input values are expected to be satisfied by other
@@ -444,12 +471,17 @@ class Task(JSONLike):
444
471
  is_multiple=True,
445
472
  parent_ref="task_template",
446
473
  ),
474
+ ChildObjectSpec(
475
+ name="output_labels",
476
+ class_name="OutputLabel",
477
+ is_multiple=True,
478
+ ),
447
479
  )
448
480
 
449
481
  def __init__(
450
482
  self,
451
483
  schemas: Union[app.TaskSchema, str, List[app.TaskSchema], List[str]],
452
- repeats: Optional[Union[int, List[int]]] = None,
484
+ repeats: Optional[List[Dict]] = None,
453
485
  groups: Optional[List[app.ElementGroup]] = None,
454
486
  resources: Optional[Dict[str, Dict]] = None,
455
487
  inputs: Optional[List[app.InputValue]] = None,
@@ -458,6 +490,7 @@ class Task(JSONLike):
458
490
  input_sources: Optional[Dict[str, app.InputSource]] = None,
459
491
  nesting_order: Optional[List] = None,
460
492
  element_sets: Optional[List[app.ElementSet]] = None,
493
+ output_labels: Optional[List[app.OutputLabel]] = None,
461
494
  sourceable_elem_iters: Optional[List[int]] = None,
462
495
  ):
463
496
  """
@@ -515,6 +548,7 @@ class Task(JSONLike):
515
548
  element_sets=element_sets,
516
549
  sourceable_elem_iters=sourceable_elem_iters,
517
550
  )
551
+ self._output_labels = output_labels or []
518
552
 
519
553
  # appended to when new element sets are added and reset on dump to disk:
520
554
  self._pending_element_sets = []
@@ -700,6 +734,10 @@ class Task(JSONLike):
700
734
  else:
701
735
  return None
702
736
 
737
+ @property
738
+ def output_labels(self):
739
+ return self._output_labels
740
+
703
741
  @property
704
742
  def _element_indices(self):
705
743
  return self.workflow_template.workflow.tasks[self.index].element_indices
@@ -707,7 +745,7 @@ class Task(JSONLike):
707
745
  def get_available_task_input_sources(
708
746
  self,
709
747
  element_set: app.ElementSet,
710
- source_tasks: Optional[List[app.Task]] = None,
748
+ source_tasks: Optional[List[app.WorkflowTask]] = None,
711
749
  ) -> List[app.InputSource]:
712
750
  """For each input parameter of this task, generate a list of possible input sources
713
751
  that derive from inputs or outputs of this and other provided tasks.
@@ -715,8 +753,6 @@ class Task(JSONLike):
715
753
  Note this only produces a subset of available input sources for each input
716
754
  parameter; other available input sources may exist from workflow imports."""
717
755
 
718
- # TODO: also search sub-parameters in the source tasks!
719
-
720
756
  if source_tasks:
721
757
  # ensure parameters provided by later tasks are added to the available sources
722
758
  # list first, meaning they take precedence when choosing an input source:
@@ -726,30 +762,32 @@ class Task(JSONLike):
726
762
 
727
763
  available = {}
728
764
  for inputs_path, inp_status in self.get_input_statuses(element_set).items():
729
- available[inputs_path] = []
730
-
731
765
  # local specification takes precedence:
732
766
  if inputs_path in element_set.get_locally_defined_inputs():
767
+ if inputs_path not in available:
768
+ available[inputs_path] = []
733
769
  available[inputs_path].append(self.app.InputSource.local())
734
770
 
735
771
  # search for task sources:
736
- for src_task_i in source_tasks:
772
+ for src_wk_task_i in source_tasks:
737
773
  # ensure we process output types before input types, so they appear in the
738
774
  # available sources list first, meaning they take precedence when choosing
739
775
  # an input source:
740
- for param_i in sorted(
741
- src_task_i.provides_parameters,
742
- key=lambda x: x.input_or_output,
776
+ src_task_i = src_wk_task_i.template
777
+ for in_or_out, labelled_path in sorted(
778
+ src_task_i.provides_parameters(),
779
+ key=lambda x: x[0],
743
780
  reverse=True,
744
781
  ):
745
- if param_i.typ == inputs_path:
746
- if param_i.input_or_output == "input":
782
+ if inputs_path in labelled_path:
783
+ avail_src_path = labelled_path
784
+ if in_or_out == "input":
747
785
  # input parameter might not be provided e.g. if it only used
748
786
  # to generate an input file, and that input file is passed
749
787
  # directly, so consider only source task element sets that
750
788
  # provide the input locally:
751
789
  es_idx = src_task_i.get_param_provided_element_sets(
752
- param_i.typ
790
+ labelled_path
753
791
  )
754
792
  else:
755
793
  # outputs are always available, so consider all source task
@@ -774,17 +812,69 @@ class Task(JSONLike):
774
812
  set(element_set.sourceable_elem_iters)
775
813
  & set(src_elem_iters)
776
814
  )
777
- if not src_elem_iters:
778
- continue
779
815
 
780
- task_source = self.app.InputSource.task(
781
- task_ref=src_task_i.insert_ID,
782
- task_source_type=param_i.input_or_output,
783
- element_iters=src_elem_iters,
784
- )
785
- available[inputs_path].append(task_source)
816
+ elif labelled_path in inputs_path and in_or_out == "output":
817
+ avail_src_path = inputs_path
818
+
819
+ inputs_path_label = None
820
+ out_label = None
821
+ try:
822
+ inputs_path_label = inputs_path.split("[")[1].split("]")[0]
823
+ except IndexError:
824
+ pass
825
+ if inputs_path_label:
826
+ for out_lab_i in src_task_i.output_labels:
827
+ if out_lab_i.label == inputs_path_label:
828
+ out_label = out_lab_i
829
+ if out_label:
830
+ # find element iteration IDs that match the output label
831
+ # filter:
832
+ param_path = ".".join(
833
+ i.condition.callable.kwargs["value"]
834
+ for i in out_label.where.path
835
+ )
836
+ param_path_split = param_path.split(".")
837
+
838
+ src_elem_iters = []
839
+ for elem_i in src_wk_task_i.elements[:]:
840
+ params = getattr(elem_i, param_path_split[0])
841
+ param_dat = getattr(params, param_path_split[1]).value
842
+
843
+ # for remaining paths components try both getattr and getitem:
844
+ for path_k in param_path_split[2:]:
845
+ try:
846
+ param_dat = param_dat[path_k]
847
+ except TypeError:
848
+ param_dat = getattr(param_dat, path_k)
849
+
850
+ rule = Rule(
851
+ path=[0],
852
+ condition=out_label.where.condition,
853
+ cast=out_label.where.cast,
854
+ )
855
+ if rule.test([param_dat]).is_valid:
856
+ src_elem_iters.append(elem_i.iterations[0].id_)
857
+
858
+ else:
859
+ continue
860
+
861
+ if not src_elem_iters:
862
+ continue
863
+
864
+ task_source = self.app.InputSource.task(
865
+ task_ref=src_task_i.insert_ID,
866
+ task_source_type=in_or_out,
867
+ element_iters=src_elem_iters,
868
+ )
869
+
870
+ if avail_src_path not in available:
871
+ available[avail_src_path] = []
872
+
873
+ available[avail_src_path].append(task_source)
786
874
 
787
875
  if inp_status.has_default:
876
+ if inputs_path not in available:
877
+ available[inputs_path] = []
788
878
  available[inputs_path].append(self.app.InputSource.default())
789
879
 
790
880
  return available
@@ -885,18 +975,20 @@ class Task(JSONLike):
885
975
 
886
976
  provided_files = [i.file for i in element_set.input_files]
887
977
  for schema in self.schemas:
978
+ if not schema.actions:
979
+ return True # for empty tasks that are used merely for defining inputs
888
980
  for act in schema.actions:
889
981
  if act.is_input_type_required(typ, provided_files):
890
982
  return True
891
983
 
892
984
  return False
893
985
 
894
- def get_param_provided_element_sets(self, typ: str) -> List[int]:
986
+ def get_param_provided_element_sets(self, labelled_path: str) -> List[int]:
895
987
  """Get the element set indices of this task for which a specified parameter type
896
988
  is locally provided."""
897
989
  es_idx = []
898
990
  for idx, src_es in enumerate(self.element_sets):
899
- if src_es.is_input_type_provided(typ):
991
+ if src_es.is_input_type_provided(labelled_path):
900
992
  es_idx.append(idx)
901
993
  return es_idx
902
994
 
@@ -913,19 +1005,18 @@ class Task(JSONLike):
913
1005
 
914
1006
  status = {}
915
1007
  for schema_input in self.all_schema_inputs:
916
- typ = schema_input.parameter.typ
917
- status[typ] = InputStatus(
918
- has_default=schema_input.default_value is not None,
919
- is_provided=elem_set.is_input_type_provided(typ),
920
- is_required=self.is_input_type_required(typ, elem_set),
921
- )
1008
+ for lab_info in schema_input.labelled_info():
1009
+ labelled_type = lab_info["labelled_type"]
1010
+ status[labelled_type] = InputStatus(
1011
+ has_default="default_value" in lab_info,
1012
+ is_provided=elem_set.is_input_type_provided(labelled_type),
1013
+ is_required=self.is_input_type_required(labelled_type, elem_set),
1014
+ )
922
1015
 
923
1016
  for inp_path in elem_set.get_defined_sub_parameter_types():
924
1017
  root_param = ".".join(inp_path.split(".")[:-1])
925
- # Sub-parameters can only be specified locally (currently), so "is_provided"
926
- # is True by definition, and if the root parameter is required then the
927
- # sub-parameter should also be required, otherwise there would be no point in
928
- # specifying it:
1018
+ # If the root parameter is required then the sub-parameter should also be
1019
+ # required, otherwise there would be no point in specifying it:
929
1020
  status[inp_path] = InputStatus(
930
1021
  has_default=False,
931
1022
  is_provided=True,
@@ -934,19 +1025,15 @@ class Task(JSONLike):
934
1025
 
935
1026
  return status
936
1027
 
937
- def get_all_required_schema_inputs(self, element_set):
938
- stats = self.get_input_statuses(element_set)
939
- return tuple(
940
- i for i in self.all_schema_inputs if stats[i.parameter.typ].is_required
941
- )
942
-
943
1028
  @property
944
1029
  def universal_input_types(self):
945
1030
  """Get input types that are associated with all schemas"""
1031
+ raise NotImplementedError()
946
1032
 
947
1033
  @property
948
1034
  def non_universal_input_types(self):
949
1035
  """Get input types for each schema that are non-universal."""
1036
+ raise NotImplementedError()
950
1037
 
951
1038
  @property
952
1039
  def defined_input_types(self):
@@ -970,15 +1057,26 @@ class Task(JSONLike):
970
1057
  """Get schema input types for which no input sources are currently specified."""
971
1058
  return self.all_schema_input_types - set(self.input_sources.keys())
972
1059
 
973
- @property
974
- def provides_parameters(self):
975
- return tuple(j for schema in self.schemas for j in schema.provides_parameters)
1060
+ def provides_parameters(self) -> Tuple[Tuple[str, str]]:
1061
+ """Get all provided parameter labelled types and whether they are inputs and
1062
+ outputs, considering all element sets.
1063
+
1064
+ """
1065
+ out = []
1066
+ for schema in self.schemas:
1067
+ for in_or_out, labelled_type in schema.provides_parameters:
1068
+ out.append((in_or_out, labelled_type))
976
1069
 
977
- def get_sub_parameter_input_values(self):
978
- return [i for i in self.inputs if i.is_sub_value]
1070
+ # add sub-parameter input values and sequences:
1071
+ for es_i in self.element_sets:
1072
+ for inp_j in es_i.inputs:
1073
+ if inp_j.is_sub_value:
1074
+ out.append(("input", inp_j.normalised_inputs_path))
1075
+ for seq_j in es_i.sequences:
1076
+ if seq_j.is_sub_value:
1077
+ out.append(("input", seq_j.normalised_inputs_path))
979
1078
 
980
- def get_non_sub_parameter_input_values(self):
981
- return [i for i in self.inputs if not i.is_sub_value]
1079
+ return tuple(out)
982
1080
 
983
1081
  def add_group(
984
1082
  self, name: str, where: app.ElementFilter, group_by_distinct: app.ParameterPath
@@ -1099,13 +1197,6 @@ class WorkflowTask:
1099
1197
  sequence_idx = {}
1100
1198
  source_idx = {}
1101
1199
 
1102
- # from pprint import pprint
1103
-
1104
- # print(f"{element_set.groups=}")
1105
- # print(f"{element_set.inputs=}")
1106
- # print("element_set.input_sources")
1107
- # pprint(element_set.input_sources)
1108
-
1109
1200
  # Assign first assuming all locally defined values are to be used:
1110
1201
  param_src = {
1111
1202
  "type": "local_input",
@@ -1151,25 +1242,39 @@ class WorkflowTask:
1151
1242
  except ValueError:
1152
1243
  pass
1153
1244
 
1245
+ for rep_spec in element_set.repeats:
1246
+ seq_key = f"repeats.{rep_spec['name']}"
1247
+ num_range = list(range(rep_spec["number"]))
1248
+ input_data_idx[seq_key] = num_range
1249
+ sequence_idx[seq_key] = num_range
1250
+
1154
1251
  # Now check for task- and default-sources and overwrite or append to local sources:
1155
- for schema_input in self.template.get_all_required_schema_inputs(element_set):
1156
- key = f"inputs.{schema_input.typ}"
1157
- sources = element_set.input_sources[schema_input.typ]
1252
+ inp_stats = self.template.get_input_statuses(element_set)
1253
+ for labelled_path_i, sources_i in element_set.input_sources.items():
1254
+ path_i_split = labelled_path_i.split(".")
1255
+ is_path_i_sub = len(path_i_split) > 1
1256
+ if is_path_i_sub:
1257
+ path_i_root = path_i_split[0]
1258
+ else:
1259
+ path_i_root = labelled_path_i
1260
+ if not inp_stats[path_i_root].is_required:
1261
+ continue
1158
1262
 
1159
- inp_group_name = schema_input.group
1160
- # print(f"{inp_group_name=}")
1263
+ inp_group_name, def_val = None, None
1264
+ for schema_input in self.template.all_schema_inputs:
1265
+ for lab_info in schema_input.labelled_info():
1266
+ if lab_info["labelled_type"] == path_i_root:
1267
+ inp_group_name = lab_info["group"]
1268
+ if "default_value" in lab_info:
1269
+ def_val = lab_info["default_value"]
1270
+ break
1161
1271
 
1162
- for inp_src_idx, inp_src in enumerate(sources):
1163
- if inp_src.source_type is InputSourceType.TASK:
1164
- # print(f" inp_src: {inp_src}")
1272
+ key = f"inputs.{labelled_path_i}"
1165
1273
 
1274
+ for inp_src_idx, inp_src in enumerate(sources_i):
1275
+ if inp_src.source_type is InputSourceType.TASK:
1166
1276
  src_task = inp_src.get_task(self.workflow)
1167
-
1168
- # print(f" src_task: {src_task}")
1169
- # print(f" {src_task.template.element_sets[0].groups=}")
1170
-
1171
1277
  src_elem_iters = src_task.get_all_element_iterations()
1172
- # print(f" {src_elem_iters=}")
1173
1278
 
1174
1279
  if inp_src.element_iters:
1175
1280
  # only include "sourceable" element iterations:
@@ -1180,34 +1285,70 @@ class WorkflowTask:
1180
1285
  src_elem_set_idx = [
1181
1286
  i.element.element_set_idx for i in src_elem_iters
1182
1287
  ]
1183
- # print(f" {src_elem_set_idx=}")
1184
1288
 
1185
1289
  if not src_elem_iters:
1186
1290
  continue
1187
1291
 
1188
1292
  task_source_type = inp_src.task_source_type.name.lower()
1189
- src_key = f"{task_source_type}s.{schema_input.typ}"
1293
+ if task_source_type == "output" and "[" in labelled_path_i:
1294
+ src_key = f"{task_source_type}s.{labelled_path_i.split('[')[0]}"
1295
+ else:
1296
+ src_key = f"{task_source_type}s.{labelled_path_i}"
1297
+
1190
1298
  grp_idx = [
1191
1299
  iter_i.get_data_idx()[src_key] for iter_i in src_elem_iters
1192
1300
  ]
1193
1301
 
1194
- # print(f" grp_idx (1): {grp_idx}")
1195
-
1196
1302
  if inp_group_name:
1197
- # print(
1198
- # f" only use elements which are part of a group definition named: {inp_group_name}."
1199
- # )
1200
1303
  group_dat_idx = []
1201
- for dat_idx_i, src_set_idx_i in zip(grp_idx, src_elem_set_idx):
1304
+ for dat_idx_i, src_set_idx_i, src_iter in zip(
1305
+ grp_idx, src_elem_set_idx, src_elem_iters
1306
+ ):
1202
1307
  src_es = src_task.template.element_sets[src_set_idx_i]
1203
1308
  if inp_group_name in [i.name for i in src_es.groups or []]:
1204
1309
  # print(f"IN GROUP; {dat_idx_i}; {src_set_idx_i=}")
1205
1310
  group_dat_idx.append(dat_idx_i)
1311
+ else:
1312
+ # if for any recursive iteration dependency, this group is
1313
+ # defined, assign:
1314
+ src_iter_i = src_iter
1315
+ src_iter_deps = (
1316
+ self.workflow.get_element_iterations_from_IDs(
1317
+ src_iter_i.get_element_iteration_dependencies(),
1318
+ )
1319
+ )
1320
+
1321
+ src_iter_deps_groups = [
1322
+ j
1323
+ for i in src_iter_deps
1324
+ for j in i.element.element_set.groups
1325
+ ]
1326
+
1327
+ if inp_group_name in [
1328
+ i.name for i in src_iter_deps_groups or []
1329
+ ]:
1330
+ group_dat_idx.append(dat_idx_i)
1331
+
1332
+ # also check input dependencies
1333
+ for (
1334
+ k,
1335
+ v,
1336
+ ) in src_iter.element.get_input_dependencies().items():
1337
+ k_es_idx = v["element_set_idx"]
1338
+ k_task_iID = v["task_insert_ID"]
1339
+ k_es = self.workflow.tasks.get(
1340
+ insert_ID=k_task_iID
1341
+ ).template.element_sets[k_es_idx]
1342
+ if inp_group_name in [
1343
+ i.name for i in k_es.groups or []
1344
+ ]:
1345
+ group_dat_idx.append(dat_idx_i)
1346
+
1347
+ # TODO: this only goes to one level of dependency
1206
1348
 
1207
1349
  grp_idx = [group_dat_idx] # TODO: generalise to multiple groups
1208
- # print(f" grp_idx (2): {grp_idx}")
1209
1350
 
1210
- if self.app.InputSource.local() in sources:
1351
+ if self.app.InputSource.local() in sources_i:
1211
1352
  # add task source to existing local source:
1212
1353
  input_data_idx[key] += grp_idx
1213
1354
  source_idx[key] += [inp_src_idx] * len(grp_idx)
@@ -1221,8 +1362,8 @@ class WorkflowTask:
1221
1362
  seq = element_set.get_sequence_by_path(key)
1222
1363
 
1223
1364
  elif inp_src.source_type is InputSourceType.DEFAULT:
1224
- grp_idx = [schema_input.default_value._value_group_idx]
1225
- if self.app.InputSource.local() in sources:
1365
+ grp_idx = [def_val._value_group_idx]
1366
+ if self.app.InputSource.local() in sources_i:
1226
1367
  input_data_idx[key] += grp_idx
1227
1368
  source_idx[key] += [inp_src_idx] * len(grp_idx)
1228
1369
 
@@ -1232,19 +1373,23 @@ class WorkflowTask:
1232
1373
 
1233
1374
  # sort smallest to largest path, so more-specific items overwrite less-specific
1234
1375
  # items parameter retrieval:
1235
- # TODO: is this still necessary?
1376
+ input_data_idx = dict(sorted(input_data_idx.items()))
1236
1377
 
1237
1378
  return (input_data_idx, sequence_idx, source_idx)
1238
1379
 
1239
1380
  def ensure_input_sources(self, element_set):
1240
1381
  """Check valid input sources are specified for a new task to be added to the
1241
1382
  workflow in a given position. If none are specified, set them according to the
1242
- default behaviour."""
1383
+ default behaviour.
1243
1384
 
1244
- # this just depends on this schema and other schemas:
1385
+ This method mutates `element_set.input_sources`.
1386
+
1387
+ """
1388
+
1389
+ # this depends on this schema, other task schemas and inputs/sequences:
1245
1390
  available_sources = self.template.get_available_task_input_sources(
1246
1391
  element_set=element_set,
1247
- source_tasks=self.workflow.template.tasks[: self.index],
1392
+ source_tasks=self.workflow.tasks[: self.index],
1248
1393
  )
1249
1394
 
1250
1395
  unreq_sources = set(element_set.input_sources.keys()) - set(
@@ -1264,33 +1409,40 @@ class WorkflowTask:
1264
1409
 
1265
1410
  all_stats = self.template.get_input_statuses(element_set)
1266
1411
 
1412
+ # an input is not required if it is only used to generate an input file that is
1413
+ # passed directly:
1414
+ req_types = set(k for k, v in all_stats.items() if v.is_required)
1415
+
1267
1416
  # check any specified sources are valid, and replace them with those computed in
1268
- # available_sources since this will have `element_iters` assigned:
1269
- for inputs_path in all_stats:
1417
+ # `available_sources` since these will have `element_iters` assigned:
1418
+ for path_i, avail_i in available_sources.items():
1419
+ # for each sub-path in available sources, if the "root-path" source is
1420
+ # required, then add the sub-path source to `req_types` as well:
1421
+ path_i_split = path_i.split(".")
1422
+ is_path_i_sub = len(path_i_split) > 1
1423
+ if is_path_i_sub:
1424
+ path_i_root = path_i_split[0]
1425
+ if path_i_root in req_types:
1426
+ req_types.add(path_i)
1427
+
1270
1428
  for s_idx, specified_source in enumerate(
1271
- element_set.input_sources.get(inputs_path, [])
1429
+ element_set.input_sources.get(path_i, [])
1272
1430
  ):
1273
1431
  self.workflow._resolve_input_source_task_reference(
1274
1432
  specified_source, self.unique_name
1275
1433
  )
1276
- avail_idx = specified_source.is_in(available_sources[inputs_path])
1434
+ avail_idx = specified_source.is_in(avail_i)
1277
1435
  if avail_idx is None:
1278
1436
  raise ValueError(
1279
1437
  f"The input source {specified_source.to_string()!r} is not "
1280
- f"available for input path {inputs_path!r}. Available "
1281
- f"input sources are: "
1282
- f"{[i.to_string() for i in available_sources[inputs_path]]}"
1438
+ f"available for input path {path_i!r}. Available "
1439
+ f"input sources are: {[i.to_string() for i in avail_i]}."
1283
1440
  )
1284
1441
  else:
1285
1442
  # overwrite with the source from available_sources, since it will have
1286
1443
  # the `element_iters` attribute assigned:
1287
- element_set.input_sources[inputs_path][s_idx] = available_sources[
1288
- inputs_path
1289
- ][avail_idx]
1444
+ element_set.input_sources[path_i][s_idx] = avail_i[avail_idx]
1290
1445
 
1291
- # an input is not required if it is only used to generate an input file that is
1292
- # passed directly:
1293
- req_types = set(k for k, v in all_stats.items() if v.is_required)
1294
1446
  unsourced_inputs = req_types - set(element_set.input_sources.keys())
1295
1447
 
1296
1448
  extra_types = set(k for k, v in all_stats.items() if v.is_extra)
@@ -1307,7 +1459,8 @@ class WorkflowTask:
1307
1459
  # set source for any unsourced inputs:
1308
1460
  missing = []
1309
1461
  for input_type in unsourced_inputs:
1310
- inp_i_sources = available_sources[input_type]
1462
+ inp_i_sources = available_sources.get(input_type, [])
1463
+
1311
1464
  source = None
1312
1465
  try:
1313
1466
  # first element is defined by default to take precedence in
@@ -1336,6 +1489,29 @@ class WorkflowTask:
1336
1489
  # element iterations for which all parameters are available (the set
1337
1490
  # intersection):
1338
1491
  for sources in sources_by_task.values():
1492
+ # if a parameter has multiple labels, disregard from this by removing all
1493
+ # parameters:
1494
+ seen_labelled = {}
1495
+ for src_i in sources.keys():
1496
+ if "[" in src_i:
1497
+ unlabelled = src_i.split("[")[0]
1498
+ if unlabelled not in seen_labelled:
1499
+ seen_labelled[unlabelled] = 1
1500
+ else:
1501
+ seen_labelled[unlabelled] += 1
1502
+
1503
+ for unlabelled, count in seen_labelled.items():
1504
+ if count > 1:
1505
+ # remove:
1506
+ sources = {
1507
+ k: v
1508
+ for k, v in sources.items()
1509
+ if not k.startswith(unlabelled)
1510
+ }
1511
+
1512
+ if len(sources) < 2:
1513
+ continue
1514
+
1339
1515
  first_src = next(iter(sources.values()))
1340
1516
  intersect_task_i = set(first_src.element_iters)
1341
1517
  for src_i in sources.values():
@@ -1537,7 +1713,6 @@ class WorkflowTask:
1537
1713
 
1538
1714
  """
1539
1715
 
1540
- # print()
1541
1716
  self.template.set_sequence_parameters(element_set)
1542
1717
 
1543
1718
  self.ensure_input_sources(element_set) # may modify element_set.input_sources
@@ -1547,29 +1722,14 @@ class WorkflowTask:
1547
1722
  element_set_idx=self.num_element_sets,
1548
1723
  )
1549
1724
 
1550
- # from pprint import pprint
1551
-
1552
- # print("input_data_idx")
1553
- # pprint(input_data_idx)
1554
-
1555
1725
  element_set.task_template = self.template # may modify element_set.nesting_order
1556
1726
 
1557
1727
  multiplicities = self.template.prepare_element_resolution(
1558
1728
  element_set, input_data_idx
1559
1729
  )
1560
1730
 
1561
- # print("multiplicities")
1562
- # pprint(multiplicities)
1563
-
1564
1731
  element_inp_data_idx = self.resolve_element_data_indices(multiplicities)
1565
1732
 
1566
- # print("element_inp_data_idx")
1567
- # pprint(element_inp_data_idx)
1568
-
1569
- # global_element_iter_idx_range = [
1570
- # self.workflow.num_element_iterations,
1571
- # self.workflow.num_element_iterations + len(element_inp_data_idx),
1572
- # ]
1573
1733
  local_element_idx_range = [
1574
1734
  self.num_elements,
1575
1735
  self.num_elements + len(element_inp_data_idx),
@@ -1583,9 +1743,6 @@ class WorkflowTask:
1583
1743
  local_element_idx_range=local_element_idx_range,
1584
1744
  )
1585
1745
 
1586
- # print("output_data_idx")
1587
- # pprint(output_data_idx)
1588
-
1589
1746
  (element_data_idx, element_seq_idx, element_src_idx) = self.generate_new_elements(
1590
1747
  input_data_idx,
1591
1748
  output_data_idx,
@@ -1593,21 +1750,11 @@ class WorkflowTask:
1593
1750
  seq_idx,
1594
1751
  src_idx,
1595
1752
  )
1596
- # print("element_data_idx")
1597
- # pprint(element_data_idx)
1598
1753
 
1599
1754
  iter_IDs = []
1600
1755
  elem_IDs = []
1601
1756
  for elem_idx, data_idx in enumerate(element_data_idx):
1602
1757
  schema_params = set(i for i in data_idx.keys() if len(i.split(".")) == 2)
1603
- # elements.append(
1604
- # {
1605
- # "iterations_idx": [self.num_elements + elem_idx],
1606
- # "es_idx": self.num_element_sets - 1,
1607
- # "seq_idx": ,
1608
- # "src_idx": ,
1609
- # }
1610
- # )
1611
1758
  elem_ID_i = self.workflow._store.add_element(
1612
1759
  task_ID=self.insert_ID,
1613
1760
  es_idx=self.num_element_sets - 1,
@@ -1622,27 +1769,7 @@ class WorkflowTask:
1622
1769
  iter_IDs.append(iter_ID_i)
1623
1770
  elem_IDs.append(elem_ID_i)
1624
1771
 
1625
- # element_iterations.append(
1626
- # {
1627
- # "global_idx": self.workflow.num_element_iterations + elem_idx,
1628
- # "data_idx": data_idx,
1629
- # "EARs_initialised": False,
1630
- # "actions": {},
1631
- # "schema_parameters": list(schema_params),
1632
- # "loop_idx": {},
1633
- # }
1634
- # )
1635
-
1636
- # self.workflow._store.add_elements(
1637
- # self.index,
1638
- # self.insert_ID,
1639
- # elements,
1640
- # element_iterations,
1641
- # )
1642
1772
  self._pending_element_IDs += elem_IDs
1643
- # self._pending_num_elements += len(element_data_idx)
1644
- # self._pending_num_element_iterations += len(element_data_idx)
1645
-
1646
1773
  self.initialise_EARs()
1647
1774
 
1648
1775
  return iter_IDs
@@ -1895,9 +2022,13 @@ class WorkflowTask:
1895
2022
  return
1896
2023
 
1897
2024
  if path[0] == "inputs":
2025
+ try:
2026
+ path_1 = path[1].split("[")[0]
2027
+ except IndexError:
2028
+ path_1 = path[1]
1898
2029
  for i in self.template.schemas:
1899
2030
  for j in i.inputs:
1900
- if j.parameter.typ == path[1]:
2031
+ if j.parameter.typ == path_1:
1901
2032
  return j.parameter
1902
2033
 
1903
2034
  elif path[0] == "outputs":
@@ -1950,29 +2081,33 @@ class WorkflowTask:
1950
2081
 
1951
2082
  data = []
1952
2083
  for data_idx_i_j in data_idx_i:
1953
- param_j = self.workflow.get_parameter(data_idx_i_j)
1954
- if param_j.file:
1955
- if param_j.file["store_contents"]:
1956
- data_j = Path(self.workflow.path) / param_j.file["path"]
1957
- else:
1958
- data_j = Path(param_j.file["path"])
1959
- data_j = data_j.as_posix()
2084
+ if path_i[0] == "repeats":
2085
+ # data is an integer repeats index, rather than a parameter ID:
2086
+ data_j = data_idx_i_j
1960
2087
  else:
1961
- data_j = param_j.data
1962
- if parameter and len(path_i) == 2:
1963
- # retrieve the source if this is a non-sub parameter, so we can,
1964
- # in the case that there is an associated `ParameterValue` class,
1965
- # get the class method that should be invoked to initialise the
1966
- # object:
1967
- if is_multi:
1968
- sources.append(param_j.source)
2088
+ param_j = self.workflow.get_parameter(data_idx_i_j)
2089
+ if param_j.file:
2090
+ if param_j.file["store_contents"]:
2091
+ data_j = Path(self.workflow.path) / param_j.file["path"]
1969
2092
  else:
1970
- sources = param_j.source
1971
- if raise_on_unset and not param_j.is_set:
1972
- raise UnsetParameterDataError(
1973
- f"Element data path {path!r} resolves to unset data for (at "
1974
- f"least) data index path: {path_i!r}."
1975
- )
2093
+ data_j = Path(param_j.file["path"])
2094
+ data_j = data_j.as_posix()
2095
+ else:
2096
+ data_j = param_j.data
2097
+ if parameter and len(path_i) == 2:
2098
+ # retrieve the source if this is a non-sub parameter, so we can,
2099
+ # in the case that there is an associated `ParameterValue` class,
2100
+ # get the class method that should be invoked to initialise the
2101
+ # object:
2102
+ if is_multi:
2103
+ sources.append(param_j.source)
2104
+ else:
2105
+ sources = param_j.source
2106
+ if raise_on_unset and not param_j.is_set:
2107
+ raise UnsetParameterDataError(
2108
+ f"Element data path {path!r} resolves to unset data for (at "
2109
+ f"least) data index path: {path_i!r}."
2110
+ )
1976
2111
  if not is_multi:
1977
2112
  data = data_j
1978
2113
  else:
@@ -2000,9 +2135,10 @@ class WorkflowTask:
2000
2135
  current_value = default
2001
2136
 
2002
2137
  if parameter and parameter._value_class:
2138
+ # TODO: retrieve value class method case-insensitively!
2003
2139
  if isinstance(current_value, dict):
2004
2140
  # return a ParameterValue instance:
2005
- method_name = sources["value_class_method"]
2141
+ method_name = sources.get("value_class_method")
2006
2142
  if method_name:
2007
2143
  method = getattr(parameter._value_class, method_name)
2008
2144
  else:
@@ -2013,7 +2149,7 @@ class WorkflowTask:
2013
2149
  # return a list of ParameterValue instances:
2014
2150
  current_value_ = []
2015
2151
  for cur_val, src in zip(current_value, sources):
2016
- method_name = src["value_class_method"]
2152
+ method_name = src.get("value_class_method")
2017
2153
  if method_name:
2018
2154
  method = getattr(parameter._value_class, method_name)
2019
2155
  else: