hpcflow-new2 0.2.0a69__py3-none-any.whl → 0.2.0a71__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/_version.py +1 -1
- hpcflow/sdk/__init__.py +1 -0
- hpcflow/sdk/core/actions.py +24 -7
- hpcflow/sdk/core/element.py +99 -25
- hpcflow/sdk/core/parameters.py +316 -69
- hpcflow/sdk/core/task.py +312 -176
- hpcflow/sdk/core/task_schema.py +27 -18
- hpcflow/sdk/core/test_utils.py +16 -1
- hpcflow/sdk/core/utils.py +18 -2
- hpcflow/sdk/core/workflow.py +15 -11
- hpcflow/tests/unit/test_app.py +1 -8
- hpcflow/tests/unit/test_input_value.py +41 -0
- hpcflow/tests/unit/test_schema_input.py +191 -0
- hpcflow/tests/unit/test_task.py +68 -23
- hpcflow/tests/unit/test_value_sequence.py +219 -0
- hpcflow/tests/unit/test_workflow.py +200 -63
- {hpcflow_new2-0.2.0a69.dist-info → hpcflow_new2-0.2.0a71.dist-info}/METADATA +1 -1
- {hpcflow_new2-0.2.0a69.dist-info → hpcflow_new2-0.2.0a71.dist-info}/RECORD +20 -18
- {hpcflow_new2-0.2.0a69.dist-info → hpcflow_new2-0.2.0a71.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a69.dist-info → hpcflow_new2-0.2.0a71.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/core/task.py
CHANGED
@@ -113,7 +113,7 @@ class ElementSet(JSONLike):
|
|
113
113
|
input_files: Optional[List[app.InputFile]] = None,
|
114
114
|
sequences: Optional[List[app.ValueSequence]] = None,
|
115
115
|
resources: Optional[Dict[str, Dict]] = None,
|
116
|
-
repeats: Optional[
|
116
|
+
repeats: Optional[List[Dict]] = None,
|
117
117
|
groups: Optional[List[app.ElementGroup]] = None,
|
118
118
|
input_sources: Optional[Dict[str, app.InputSource]] = None,
|
119
119
|
nesting_order: Optional[List] = None,
|
@@ -143,7 +143,7 @@ class ElementSet(JSONLike):
|
|
143
143
|
|
144
144
|
self.inputs = inputs or []
|
145
145
|
self.input_files = input_files or []
|
146
|
-
self.repeats = repeats
|
146
|
+
self.repeats = repeats or []
|
147
147
|
self.groups = groups or []
|
148
148
|
self.resources = resources
|
149
149
|
self.sequences = sequences or []
|
@@ -216,7 +216,7 @@ class ElementSet(JSONLike):
|
|
216
216
|
|
217
217
|
@property
|
218
218
|
def input_types(self):
|
219
|
-
return [i.
|
219
|
+
return [i.labelled_type for i in self.inputs]
|
220
220
|
|
221
221
|
@property
|
222
222
|
def element_local_idx_range(self):
|
@@ -266,7 +266,7 @@ class ElementSet(JSONLike):
|
|
266
266
|
|
267
267
|
seq_inp_types = []
|
268
268
|
for seq_i in self.sequences:
|
269
|
-
inp_type = seq_i.
|
269
|
+
inp_type = seq_i.labelled_type
|
270
270
|
if inp_type:
|
271
271
|
bad_inp = {inp_type} - self.task_template.all_schema_input_types
|
272
272
|
allowed_str = ", ".join(
|
@@ -282,6 +282,11 @@ class ElementSet(JSONLike):
|
|
282
282
|
if seq_i.path not in self.nesting_order:
|
283
283
|
self.nesting_order.update({seq_i.path: seq_i.nesting_order})
|
284
284
|
|
285
|
+
for rep_spec in self.repeats:
|
286
|
+
reps_path_i = f'repeats.{rep_spec["name"]}'
|
287
|
+
if reps_path_i not in self.nesting_order:
|
288
|
+
self.nesting_order[reps_path_i] = rep_spec["nesting_order"]
|
289
|
+
|
285
290
|
for k, v in self.nesting_order.items():
|
286
291
|
if v < 0:
|
287
292
|
raise TaskTemplateInvalidNesting(
|
@@ -408,22 +413,44 @@ class ElementSet(JSONLike):
|
|
408
413
|
|
409
414
|
return deps
|
410
415
|
|
411
|
-
def is_input_type_provided(self,
|
416
|
+
def is_input_type_provided(self, labelled_path: str) -> bool:
|
412
417
|
"""Check if an input is provided locally as an InputValue or a ValueSequence."""
|
413
418
|
|
414
419
|
for inp in self.inputs:
|
415
|
-
if
|
420
|
+
if labelled_path == inp.normalised_inputs_path:
|
416
421
|
return True
|
417
422
|
|
418
423
|
for seq in self.sequences:
|
419
424
|
if seq.parameter:
|
420
425
|
# i.e. not a resource:
|
421
|
-
if
|
426
|
+
if labelled_path == seq.normalised_inputs_path:
|
422
427
|
return True
|
423
428
|
|
424
429
|
return False
|
425
430
|
|
426
431
|
|
432
|
+
class OutputLabel(JSONLike):
|
433
|
+
"""Class to represent schema input labels that should be applied to a subset of task
|
434
|
+
outputs"""
|
435
|
+
|
436
|
+
_child_objects = (
|
437
|
+
ChildObjectSpec(
|
438
|
+
name="where",
|
439
|
+
class_name="ElementFilter",
|
440
|
+
),
|
441
|
+
)
|
442
|
+
|
443
|
+
def __init__(
|
444
|
+
self,
|
445
|
+
parameter: str,
|
446
|
+
label: str,
|
447
|
+
where: Optional[List[app.ElementFilter]] = None,
|
448
|
+
) -> None:
|
449
|
+
self.parameter = parameter
|
450
|
+
self.label = label
|
451
|
+
self.where = where
|
452
|
+
|
453
|
+
|
427
454
|
class Task(JSONLike):
|
428
455
|
"""Parametrisation of an isolated task for which a subset of input values are given
|
429
456
|
"locally". The remaining input values are expected to be satisfied by other
|
@@ -444,12 +471,17 @@ class Task(JSONLike):
|
|
444
471
|
is_multiple=True,
|
445
472
|
parent_ref="task_template",
|
446
473
|
),
|
474
|
+
ChildObjectSpec(
|
475
|
+
name="output_labels",
|
476
|
+
class_name="OutputLabel",
|
477
|
+
is_multiple=True,
|
478
|
+
),
|
447
479
|
)
|
448
480
|
|
449
481
|
def __init__(
|
450
482
|
self,
|
451
483
|
schemas: Union[app.TaskSchema, str, List[app.TaskSchema], List[str]],
|
452
|
-
repeats: Optional[
|
484
|
+
repeats: Optional[List[Dict]] = None,
|
453
485
|
groups: Optional[List[app.ElementGroup]] = None,
|
454
486
|
resources: Optional[Dict[str, Dict]] = None,
|
455
487
|
inputs: Optional[List[app.InputValue]] = None,
|
@@ -458,6 +490,7 @@ class Task(JSONLike):
|
|
458
490
|
input_sources: Optional[Dict[str, app.InputSource]] = None,
|
459
491
|
nesting_order: Optional[List] = None,
|
460
492
|
element_sets: Optional[List[app.ElementSet]] = None,
|
493
|
+
output_labels: Optional[List[app.OutputLabel]] = None,
|
461
494
|
sourceable_elem_iters: Optional[List[int]] = None,
|
462
495
|
):
|
463
496
|
"""
|
@@ -515,6 +548,7 @@ class Task(JSONLike):
|
|
515
548
|
element_sets=element_sets,
|
516
549
|
sourceable_elem_iters=sourceable_elem_iters,
|
517
550
|
)
|
551
|
+
self._output_labels = output_labels or []
|
518
552
|
|
519
553
|
# appended to when new element sets are added and reset on dump to disk:
|
520
554
|
self._pending_element_sets = []
|
@@ -700,6 +734,10 @@ class Task(JSONLike):
|
|
700
734
|
else:
|
701
735
|
return None
|
702
736
|
|
737
|
+
@property
|
738
|
+
def output_labels(self):
|
739
|
+
return self._output_labels
|
740
|
+
|
703
741
|
@property
|
704
742
|
def _element_indices(self):
|
705
743
|
return self.workflow_template.workflow.tasks[self.index].element_indices
|
@@ -707,7 +745,7 @@ class Task(JSONLike):
|
|
707
745
|
def get_available_task_input_sources(
|
708
746
|
self,
|
709
747
|
element_set: app.ElementSet,
|
710
|
-
source_tasks: Optional[List[app.
|
748
|
+
source_tasks: Optional[List[app.WorkflowTask]] = None,
|
711
749
|
) -> List[app.InputSource]:
|
712
750
|
"""For each input parameter of this task, generate a list of possible input sources
|
713
751
|
that derive from inputs or outputs of this and other provided tasks.
|
@@ -715,8 +753,6 @@ class Task(JSONLike):
|
|
715
753
|
Note this only produces a subset of available input sources for each input
|
716
754
|
parameter; other available input sources may exist from workflow imports."""
|
717
755
|
|
718
|
-
# TODO: also search sub-parameters in the source tasks!
|
719
|
-
|
720
756
|
if source_tasks:
|
721
757
|
# ensure parameters provided by later tasks are added to the available sources
|
722
758
|
# list first, meaning they take precedence when choosing an input source:
|
@@ -726,30 +762,32 @@ class Task(JSONLike):
|
|
726
762
|
|
727
763
|
available = {}
|
728
764
|
for inputs_path, inp_status in self.get_input_statuses(element_set).items():
|
729
|
-
available[inputs_path] = []
|
730
|
-
|
731
765
|
# local specification takes precedence:
|
732
766
|
if inputs_path in element_set.get_locally_defined_inputs():
|
767
|
+
if inputs_path not in available:
|
768
|
+
available[inputs_path] = []
|
733
769
|
available[inputs_path].append(self.app.InputSource.local())
|
734
770
|
|
735
771
|
# search for task sources:
|
736
|
-
for
|
772
|
+
for src_wk_task_i in source_tasks:
|
737
773
|
# ensure we process output types before input types, so they appear in the
|
738
774
|
# available sources list first, meaning they take precedence when choosing
|
739
775
|
# an input source:
|
740
|
-
|
741
|
-
|
742
|
-
|
776
|
+
src_task_i = src_wk_task_i.template
|
777
|
+
for in_or_out, labelled_path in sorted(
|
778
|
+
src_task_i.provides_parameters(),
|
779
|
+
key=lambda x: x[0],
|
743
780
|
reverse=True,
|
744
781
|
):
|
745
|
-
if
|
746
|
-
|
782
|
+
if inputs_path in labelled_path:
|
783
|
+
avail_src_path = labelled_path
|
784
|
+
if in_or_out == "input":
|
747
785
|
# input parameter might not be provided e.g. if it only used
|
748
786
|
# to generate an input file, and that input file is passed
|
749
787
|
# directly, so consider only source task element sets that
|
750
788
|
# provide the input locally:
|
751
789
|
es_idx = src_task_i.get_param_provided_element_sets(
|
752
|
-
|
790
|
+
labelled_path
|
753
791
|
)
|
754
792
|
else:
|
755
793
|
# outputs are always available, so consider all source task
|
@@ -774,17 +812,69 @@ class Task(JSONLike):
|
|
774
812
|
set(element_set.sourceable_elem_iters)
|
775
813
|
& set(src_elem_iters)
|
776
814
|
)
|
777
|
-
if not src_elem_iters:
|
778
|
-
continue
|
779
815
|
|
780
|
-
|
781
|
-
|
782
|
-
|
783
|
-
|
784
|
-
|
785
|
-
|
816
|
+
elif labelled_path in inputs_path and in_or_out == "output":
|
817
|
+
avail_src_path = inputs_path
|
818
|
+
|
819
|
+
inputs_path_label = None
|
820
|
+
out_label = None
|
821
|
+
try:
|
822
|
+
inputs_path_label = inputs_path.split("[")[1].split("]")[0]
|
823
|
+
except IndexError:
|
824
|
+
pass
|
825
|
+
if inputs_path_label:
|
826
|
+
for out_lab_i in src_task_i.output_labels:
|
827
|
+
if out_lab_i.label == inputs_path_label:
|
828
|
+
out_label = out_lab_i
|
829
|
+
if out_label:
|
830
|
+
# find element iteration IDs that match the output label
|
831
|
+
# filter:
|
832
|
+
param_path = ".".join(
|
833
|
+
i.condition.callable.kwargs["value"]
|
834
|
+
for i in out_label.where.path
|
835
|
+
)
|
836
|
+
param_path_split = param_path.split(".")
|
837
|
+
|
838
|
+
src_elem_iters = []
|
839
|
+
for elem_i in src_wk_task_i.elements[:]:
|
840
|
+
params = getattr(elem_i, param_path_split[0])
|
841
|
+
param_dat = getattr(params, param_path_split[1]).value
|
842
|
+
|
843
|
+
# for remaining paths components try both getattr and getitem:
|
844
|
+
for path_k in param_path_split[2:]:
|
845
|
+
try:
|
846
|
+
param_dat = param_dat[path_k]
|
847
|
+
except TypeError:
|
848
|
+
param_dat = getattr(param_dat, path_k)
|
849
|
+
|
850
|
+
rule = Rule(
|
851
|
+
path=[0],
|
852
|
+
condition=out_label.where.condition,
|
853
|
+
cast=out_label.where.cast,
|
854
|
+
)
|
855
|
+
if rule.test([param_dat]).is_valid:
|
856
|
+
src_elem_iters.append(elem_i.iterations[0].id_)
|
857
|
+
|
858
|
+
else:
|
859
|
+
continue
|
860
|
+
|
861
|
+
if not src_elem_iters:
|
862
|
+
continue
|
863
|
+
|
864
|
+
task_source = self.app.InputSource.task(
|
865
|
+
task_ref=src_task_i.insert_ID,
|
866
|
+
task_source_type=in_or_out,
|
867
|
+
element_iters=src_elem_iters,
|
868
|
+
)
|
869
|
+
|
870
|
+
if avail_src_path not in available:
|
871
|
+
available[avail_src_path] = []
|
872
|
+
|
873
|
+
available[avail_src_path].append(task_source)
|
786
874
|
|
787
875
|
if inp_status.has_default:
|
876
|
+
if inputs_path not in available:
|
877
|
+
available[inputs_path] = []
|
788
878
|
available[inputs_path].append(self.app.InputSource.default())
|
789
879
|
|
790
880
|
return available
|
@@ -885,18 +975,20 @@ class Task(JSONLike):
|
|
885
975
|
|
886
976
|
provided_files = [i.file for i in element_set.input_files]
|
887
977
|
for schema in self.schemas:
|
978
|
+
if not schema.actions:
|
979
|
+
return True # for empty tasks that are used merely for defining inputs
|
888
980
|
for act in schema.actions:
|
889
981
|
if act.is_input_type_required(typ, provided_files):
|
890
982
|
return True
|
891
983
|
|
892
984
|
return False
|
893
985
|
|
894
|
-
def get_param_provided_element_sets(self,
|
986
|
+
def get_param_provided_element_sets(self, labelled_path: str) -> List[int]:
|
895
987
|
"""Get the element set indices of this task for which a specified parameter type
|
896
988
|
is locally provided."""
|
897
989
|
es_idx = []
|
898
990
|
for idx, src_es in enumerate(self.element_sets):
|
899
|
-
if src_es.is_input_type_provided(
|
991
|
+
if src_es.is_input_type_provided(labelled_path):
|
900
992
|
es_idx.append(idx)
|
901
993
|
return es_idx
|
902
994
|
|
@@ -913,19 +1005,18 @@ class Task(JSONLike):
|
|
913
1005
|
|
914
1006
|
status = {}
|
915
1007
|
for schema_input in self.all_schema_inputs:
|
916
|
-
|
917
|
-
|
918
|
-
|
919
|
-
|
920
|
-
|
921
|
-
|
1008
|
+
for lab_info in schema_input.labelled_info():
|
1009
|
+
labelled_type = lab_info["labelled_type"]
|
1010
|
+
status[labelled_type] = InputStatus(
|
1011
|
+
has_default="default_value" in lab_info,
|
1012
|
+
is_provided=elem_set.is_input_type_provided(labelled_type),
|
1013
|
+
is_required=self.is_input_type_required(labelled_type, elem_set),
|
1014
|
+
)
|
922
1015
|
|
923
1016
|
for inp_path in elem_set.get_defined_sub_parameter_types():
|
924
1017
|
root_param = ".".join(inp_path.split(".")[:-1])
|
925
|
-
#
|
926
|
-
#
|
927
|
-
# sub-parameter should also be required, otherwise there would be no point in
|
928
|
-
# specifying it:
|
1018
|
+
# If the root parameter is required then the sub-parameter should also be
|
1019
|
+
# required, otherwise there would be no point in specifying it:
|
929
1020
|
status[inp_path] = InputStatus(
|
930
1021
|
has_default=False,
|
931
1022
|
is_provided=True,
|
@@ -934,19 +1025,15 @@ class Task(JSONLike):
|
|
934
1025
|
|
935
1026
|
return status
|
936
1027
|
|
937
|
-
def get_all_required_schema_inputs(self, element_set):
|
938
|
-
stats = self.get_input_statuses(element_set)
|
939
|
-
return tuple(
|
940
|
-
i for i in self.all_schema_inputs if stats[i.parameter.typ].is_required
|
941
|
-
)
|
942
|
-
|
943
1028
|
@property
|
944
1029
|
def universal_input_types(self):
|
945
1030
|
"""Get input types that are associated with all schemas"""
|
1031
|
+
raise NotImplementedError()
|
946
1032
|
|
947
1033
|
@property
|
948
1034
|
def non_universal_input_types(self):
|
949
1035
|
"""Get input types for each schema that are non-universal."""
|
1036
|
+
raise NotImplementedError()
|
950
1037
|
|
951
1038
|
@property
|
952
1039
|
def defined_input_types(self):
|
@@ -970,15 +1057,26 @@ class Task(JSONLike):
|
|
970
1057
|
"""Get schema input types for which no input sources are currently specified."""
|
971
1058
|
return self.all_schema_input_types - set(self.input_sources.keys())
|
972
1059
|
|
973
|
-
|
974
|
-
|
975
|
-
|
1060
|
+
def provides_parameters(self) -> Tuple[Tuple[str, str]]:
|
1061
|
+
"""Get all provided parameter labelled types and whether they are inputs and
|
1062
|
+
outputs, considering all element sets.
|
1063
|
+
|
1064
|
+
"""
|
1065
|
+
out = []
|
1066
|
+
for schema in self.schemas:
|
1067
|
+
for in_or_out, labelled_type in schema.provides_parameters:
|
1068
|
+
out.append((in_or_out, labelled_type))
|
976
1069
|
|
977
|
-
|
978
|
-
|
1070
|
+
# add sub-parameter input values and sequences:
|
1071
|
+
for es_i in self.element_sets:
|
1072
|
+
for inp_j in es_i.inputs:
|
1073
|
+
if inp_j.is_sub_value:
|
1074
|
+
out.append(("input", inp_j.normalised_inputs_path))
|
1075
|
+
for seq_j in es_i.sequences:
|
1076
|
+
if seq_j.is_sub_value:
|
1077
|
+
out.append(("input", seq_j.normalised_inputs_path))
|
979
1078
|
|
980
|
-
|
981
|
-
return [i for i in self.inputs if not i.is_sub_value]
|
1079
|
+
return tuple(out)
|
982
1080
|
|
983
1081
|
def add_group(
|
984
1082
|
self, name: str, where: app.ElementFilter, group_by_distinct: app.ParameterPath
|
@@ -1099,13 +1197,6 @@ class WorkflowTask:
|
|
1099
1197
|
sequence_idx = {}
|
1100
1198
|
source_idx = {}
|
1101
1199
|
|
1102
|
-
# from pprint import pprint
|
1103
|
-
|
1104
|
-
# print(f"{element_set.groups=}")
|
1105
|
-
# print(f"{element_set.inputs=}")
|
1106
|
-
# print("element_set.input_sources")
|
1107
|
-
# pprint(element_set.input_sources)
|
1108
|
-
|
1109
1200
|
# Assign first assuming all locally defined values are to be used:
|
1110
1201
|
param_src = {
|
1111
1202
|
"type": "local_input",
|
@@ -1151,25 +1242,39 @@ class WorkflowTask:
|
|
1151
1242
|
except ValueError:
|
1152
1243
|
pass
|
1153
1244
|
|
1245
|
+
for rep_spec in element_set.repeats:
|
1246
|
+
seq_key = f"repeats.{rep_spec['name']}"
|
1247
|
+
num_range = list(range(rep_spec["number"]))
|
1248
|
+
input_data_idx[seq_key] = num_range
|
1249
|
+
sequence_idx[seq_key] = num_range
|
1250
|
+
|
1154
1251
|
# Now check for task- and default-sources and overwrite or append to local sources:
|
1155
|
-
|
1156
|
-
|
1157
|
-
|
1252
|
+
inp_stats = self.template.get_input_statuses(element_set)
|
1253
|
+
for labelled_path_i, sources_i in element_set.input_sources.items():
|
1254
|
+
path_i_split = labelled_path_i.split(".")
|
1255
|
+
is_path_i_sub = len(path_i_split) > 1
|
1256
|
+
if is_path_i_sub:
|
1257
|
+
path_i_root = path_i_split[0]
|
1258
|
+
else:
|
1259
|
+
path_i_root = labelled_path_i
|
1260
|
+
if not inp_stats[path_i_root].is_required:
|
1261
|
+
continue
|
1158
1262
|
|
1159
|
-
inp_group_name =
|
1160
|
-
|
1263
|
+
inp_group_name, def_val = None, None
|
1264
|
+
for schema_input in self.template.all_schema_inputs:
|
1265
|
+
for lab_info in schema_input.labelled_info():
|
1266
|
+
if lab_info["labelled_type"] == path_i_root:
|
1267
|
+
inp_group_name = lab_info["group"]
|
1268
|
+
if "default_value" in lab_info:
|
1269
|
+
def_val = lab_info["default_value"]
|
1270
|
+
break
|
1161
1271
|
|
1162
|
-
|
1163
|
-
if inp_src.source_type is InputSourceType.TASK:
|
1164
|
-
# print(f" inp_src: {inp_src}")
|
1272
|
+
key = f"inputs.{labelled_path_i}"
|
1165
1273
|
|
1274
|
+
for inp_src_idx, inp_src in enumerate(sources_i):
|
1275
|
+
if inp_src.source_type is InputSourceType.TASK:
|
1166
1276
|
src_task = inp_src.get_task(self.workflow)
|
1167
|
-
|
1168
|
-
# print(f" src_task: {src_task}")
|
1169
|
-
# print(f" {src_task.template.element_sets[0].groups=}")
|
1170
|
-
|
1171
1277
|
src_elem_iters = src_task.get_all_element_iterations()
|
1172
|
-
# print(f" {src_elem_iters=}")
|
1173
1278
|
|
1174
1279
|
if inp_src.element_iters:
|
1175
1280
|
# only include "sourceable" element iterations:
|
@@ -1180,34 +1285,70 @@ class WorkflowTask:
|
|
1180
1285
|
src_elem_set_idx = [
|
1181
1286
|
i.element.element_set_idx for i in src_elem_iters
|
1182
1287
|
]
|
1183
|
-
# print(f" {src_elem_set_idx=}")
|
1184
1288
|
|
1185
1289
|
if not src_elem_iters:
|
1186
1290
|
continue
|
1187
1291
|
|
1188
1292
|
task_source_type = inp_src.task_source_type.name.lower()
|
1189
|
-
|
1293
|
+
if task_source_type == "output" and "[" in labelled_path_i:
|
1294
|
+
src_key = f"{task_source_type}s.{labelled_path_i.split('[')[0]}"
|
1295
|
+
else:
|
1296
|
+
src_key = f"{task_source_type}s.{labelled_path_i}"
|
1297
|
+
|
1190
1298
|
grp_idx = [
|
1191
1299
|
iter_i.get_data_idx()[src_key] for iter_i in src_elem_iters
|
1192
1300
|
]
|
1193
1301
|
|
1194
|
-
# print(f" grp_idx (1): {grp_idx}")
|
1195
|
-
|
1196
1302
|
if inp_group_name:
|
1197
|
-
# print(
|
1198
|
-
# f" only use elements which are part of a group definition named: {inp_group_name}."
|
1199
|
-
# )
|
1200
1303
|
group_dat_idx = []
|
1201
|
-
for dat_idx_i, src_set_idx_i in zip(
|
1304
|
+
for dat_idx_i, src_set_idx_i, src_iter in zip(
|
1305
|
+
grp_idx, src_elem_set_idx, src_elem_iters
|
1306
|
+
):
|
1202
1307
|
src_es = src_task.template.element_sets[src_set_idx_i]
|
1203
1308
|
if inp_group_name in [i.name for i in src_es.groups or []]:
|
1204
1309
|
# print(f"IN GROUP; {dat_idx_i}; {src_set_idx_i=}")
|
1205
1310
|
group_dat_idx.append(dat_idx_i)
|
1311
|
+
else:
|
1312
|
+
# if for any recursive iteration dependency, this group is
|
1313
|
+
# defined, assign:
|
1314
|
+
src_iter_i = src_iter
|
1315
|
+
src_iter_deps = (
|
1316
|
+
self.workflow.get_element_iterations_from_IDs(
|
1317
|
+
src_iter_i.get_element_iteration_dependencies(),
|
1318
|
+
)
|
1319
|
+
)
|
1320
|
+
|
1321
|
+
src_iter_deps_groups = [
|
1322
|
+
j
|
1323
|
+
for i in src_iter_deps
|
1324
|
+
for j in i.element.element_set.groups
|
1325
|
+
]
|
1326
|
+
|
1327
|
+
if inp_group_name in [
|
1328
|
+
i.name for i in src_iter_deps_groups or []
|
1329
|
+
]:
|
1330
|
+
group_dat_idx.append(dat_idx_i)
|
1331
|
+
|
1332
|
+
# also check input dependencies
|
1333
|
+
for (
|
1334
|
+
k,
|
1335
|
+
v,
|
1336
|
+
) in src_iter.element.get_input_dependencies().items():
|
1337
|
+
k_es_idx = v["element_set_idx"]
|
1338
|
+
k_task_iID = v["task_insert_ID"]
|
1339
|
+
k_es = self.workflow.tasks.get(
|
1340
|
+
insert_ID=k_task_iID
|
1341
|
+
).template.element_sets[k_es_idx]
|
1342
|
+
if inp_group_name in [
|
1343
|
+
i.name for i in k_es.groups or []
|
1344
|
+
]:
|
1345
|
+
group_dat_idx.append(dat_idx_i)
|
1346
|
+
|
1347
|
+
# TODO: this only goes to one level of dependency
|
1206
1348
|
|
1207
1349
|
grp_idx = [group_dat_idx] # TODO: generalise to multiple groups
|
1208
|
-
# print(f" grp_idx (2): {grp_idx}")
|
1209
1350
|
|
1210
|
-
if self.app.InputSource.local() in
|
1351
|
+
if self.app.InputSource.local() in sources_i:
|
1211
1352
|
# add task source to existing local source:
|
1212
1353
|
input_data_idx[key] += grp_idx
|
1213
1354
|
source_idx[key] += [inp_src_idx] * len(grp_idx)
|
@@ -1221,8 +1362,8 @@ class WorkflowTask:
|
|
1221
1362
|
seq = element_set.get_sequence_by_path(key)
|
1222
1363
|
|
1223
1364
|
elif inp_src.source_type is InputSourceType.DEFAULT:
|
1224
|
-
grp_idx = [
|
1225
|
-
if self.app.InputSource.local() in
|
1365
|
+
grp_idx = [def_val._value_group_idx]
|
1366
|
+
if self.app.InputSource.local() in sources_i:
|
1226
1367
|
input_data_idx[key] += grp_idx
|
1227
1368
|
source_idx[key] += [inp_src_idx] * len(grp_idx)
|
1228
1369
|
|
@@ -1232,19 +1373,23 @@ class WorkflowTask:
|
|
1232
1373
|
|
1233
1374
|
# sort smallest to largest path, so more-specific items overwrite less-specific
|
1234
1375
|
# items parameter retrieval:
|
1235
|
-
|
1376
|
+
input_data_idx = dict(sorted(input_data_idx.items()))
|
1236
1377
|
|
1237
1378
|
return (input_data_idx, sequence_idx, source_idx)
|
1238
1379
|
|
1239
1380
|
def ensure_input_sources(self, element_set):
|
1240
1381
|
"""Check valid input sources are specified for a new task to be added to the
|
1241
1382
|
workflow in a given position. If none are specified, set them according to the
|
1242
|
-
default behaviour.
|
1383
|
+
default behaviour.
|
1243
1384
|
|
1244
|
-
|
1385
|
+
This method mutates `element_set.input_sources`.
|
1386
|
+
|
1387
|
+
"""
|
1388
|
+
|
1389
|
+
# this depends on this schema, other task schemas and inputs/sequences:
|
1245
1390
|
available_sources = self.template.get_available_task_input_sources(
|
1246
1391
|
element_set=element_set,
|
1247
|
-
source_tasks=self.workflow.
|
1392
|
+
source_tasks=self.workflow.tasks[: self.index],
|
1248
1393
|
)
|
1249
1394
|
|
1250
1395
|
unreq_sources = set(element_set.input_sources.keys()) - set(
|
@@ -1264,33 +1409,40 @@ class WorkflowTask:
|
|
1264
1409
|
|
1265
1410
|
all_stats = self.template.get_input_statuses(element_set)
|
1266
1411
|
|
1412
|
+
# an input is not required if it is only used to generate an input file that is
|
1413
|
+
# passed directly:
|
1414
|
+
req_types = set(k for k, v in all_stats.items() if v.is_required)
|
1415
|
+
|
1267
1416
|
# check any specified sources are valid, and replace them with those computed in
|
1268
|
-
# available_sources since
|
1269
|
-
for
|
1417
|
+
# `available_sources` since these will have `element_iters` assigned:
|
1418
|
+
for path_i, avail_i in available_sources.items():
|
1419
|
+
# for each sub-path in available sources, if the "root-path" source is
|
1420
|
+
# required, then add the sub-path source to `req_types` as well:
|
1421
|
+
path_i_split = path_i.split(".")
|
1422
|
+
is_path_i_sub = len(path_i_split) > 1
|
1423
|
+
if is_path_i_sub:
|
1424
|
+
path_i_root = path_i_split[0]
|
1425
|
+
if path_i_root in req_types:
|
1426
|
+
req_types.add(path_i)
|
1427
|
+
|
1270
1428
|
for s_idx, specified_source in enumerate(
|
1271
|
-
element_set.input_sources.get(
|
1429
|
+
element_set.input_sources.get(path_i, [])
|
1272
1430
|
):
|
1273
1431
|
self.workflow._resolve_input_source_task_reference(
|
1274
1432
|
specified_source, self.unique_name
|
1275
1433
|
)
|
1276
|
-
avail_idx = specified_source.is_in(
|
1434
|
+
avail_idx = specified_source.is_in(avail_i)
|
1277
1435
|
if avail_idx is None:
|
1278
1436
|
raise ValueError(
|
1279
1437
|
f"The input source {specified_source.to_string()!r} is not "
|
1280
|
-
f"available for input path {
|
1281
|
-
f"input sources are: "
|
1282
|
-
f"{[i.to_string() for i in available_sources[inputs_path]]}"
|
1438
|
+
f"available for input path {path_i!r}. Available "
|
1439
|
+
f"input sources are: {[i.to_string() for i in avail_i]}."
|
1283
1440
|
)
|
1284
1441
|
else:
|
1285
1442
|
# overwrite with the source from available_sources, since it will have
|
1286
1443
|
# the `element_iters` attribute assigned:
|
1287
|
-
element_set.input_sources[
|
1288
|
-
inputs_path
|
1289
|
-
][avail_idx]
|
1444
|
+
element_set.input_sources[path_i][s_idx] = avail_i[avail_idx]
|
1290
1445
|
|
1291
|
-
# an input is not required if it is only used to generate an input file that is
|
1292
|
-
# passed directly:
|
1293
|
-
req_types = set(k for k, v in all_stats.items() if v.is_required)
|
1294
1446
|
unsourced_inputs = req_types - set(element_set.input_sources.keys())
|
1295
1447
|
|
1296
1448
|
extra_types = set(k for k, v in all_stats.items() if v.is_extra)
|
@@ -1307,7 +1459,8 @@ class WorkflowTask:
|
|
1307
1459
|
# set source for any unsourced inputs:
|
1308
1460
|
missing = []
|
1309
1461
|
for input_type in unsourced_inputs:
|
1310
|
-
inp_i_sources = available_sources[
|
1462
|
+
inp_i_sources = available_sources.get(input_type, [])
|
1463
|
+
|
1311
1464
|
source = None
|
1312
1465
|
try:
|
1313
1466
|
# first element is defined by default to take precedence in
|
@@ -1336,6 +1489,29 @@ class WorkflowTask:
|
|
1336
1489
|
# element iterations for which all parameters are available (the set
|
1337
1490
|
# intersection):
|
1338
1491
|
for sources in sources_by_task.values():
|
1492
|
+
# if a parameter has multiple labels, disregard from this by removing all
|
1493
|
+
# parameters:
|
1494
|
+
seen_labelled = {}
|
1495
|
+
for src_i in sources.keys():
|
1496
|
+
if "[" in src_i:
|
1497
|
+
unlabelled = src_i.split("[")[0]
|
1498
|
+
if unlabelled not in seen_labelled:
|
1499
|
+
seen_labelled[unlabelled] = 1
|
1500
|
+
else:
|
1501
|
+
seen_labelled[unlabelled] += 1
|
1502
|
+
|
1503
|
+
for unlabelled, count in seen_labelled.items():
|
1504
|
+
if count > 1:
|
1505
|
+
# remove:
|
1506
|
+
sources = {
|
1507
|
+
k: v
|
1508
|
+
for k, v in sources.items()
|
1509
|
+
if not k.startswith(unlabelled)
|
1510
|
+
}
|
1511
|
+
|
1512
|
+
if len(sources) < 2:
|
1513
|
+
continue
|
1514
|
+
|
1339
1515
|
first_src = next(iter(sources.values()))
|
1340
1516
|
intersect_task_i = set(first_src.element_iters)
|
1341
1517
|
for src_i in sources.values():
|
@@ -1537,7 +1713,6 @@ class WorkflowTask:
|
|
1537
1713
|
|
1538
1714
|
"""
|
1539
1715
|
|
1540
|
-
# print()
|
1541
1716
|
self.template.set_sequence_parameters(element_set)
|
1542
1717
|
|
1543
1718
|
self.ensure_input_sources(element_set) # may modify element_set.input_sources
|
@@ -1547,29 +1722,14 @@ class WorkflowTask:
|
|
1547
1722
|
element_set_idx=self.num_element_sets,
|
1548
1723
|
)
|
1549
1724
|
|
1550
|
-
# from pprint import pprint
|
1551
|
-
|
1552
|
-
# print("input_data_idx")
|
1553
|
-
# pprint(input_data_idx)
|
1554
|
-
|
1555
1725
|
element_set.task_template = self.template # may modify element_set.nesting_order
|
1556
1726
|
|
1557
1727
|
multiplicities = self.template.prepare_element_resolution(
|
1558
1728
|
element_set, input_data_idx
|
1559
1729
|
)
|
1560
1730
|
|
1561
|
-
# print("multiplicities")
|
1562
|
-
# pprint(multiplicities)
|
1563
|
-
|
1564
1731
|
element_inp_data_idx = self.resolve_element_data_indices(multiplicities)
|
1565
1732
|
|
1566
|
-
# print("element_inp_data_idx")
|
1567
|
-
# pprint(element_inp_data_idx)
|
1568
|
-
|
1569
|
-
# global_element_iter_idx_range = [
|
1570
|
-
# self.workflow.num_element_iterations,
|
1571
|
-
# self.workflow.num_element_iterations + len(element_inp_data_idx),
|
1572
|
-
# ]
|
1573
1733
|
local_element_idx_range = [
|
1574
1734
|
self.num_elements,
|
1575
1735
|
self.num_elements + len(element_inp_data_idx),
|
@@ -1583,9 +1743,6 @@ class WorkflowTask:
|
|
1583
1743
|
local_element_idx_range=local_element_idx_range,
|
1584
1744
|
)
|
1585
1745
|
|
1586
|
-
# print("output_data_idx")
|
1587
|
-
# pprint(output_data_idx)
|
1588
|
-
|
1589
1746
|
(element_data_idx, element_seq_idx, element_src_idx) = self.generate_new_elements(
|
1590
1747
|
input_data_idx,
|
1591
1748
|
output_data_idx,
|
@@ -1593,21 +1750,11 @@ class WorkflowTask:
|
|
1593
1750
|
seq_idx,
|
1594
1751
|
src_idx,
|
1595
1752
|
)
|
1596
|
-
# print("element_data_idx")
|
1597
|
-
# pprint(element_data_idx)
|
1598
1753
|
|
1599
1754
|
iter_IDs = []
|
1600
1755
|
elem_IDs = []
|
1601
1756
|
for elem_idx, data_idx in enumerate(element_data_idx):
|
1602
1757
|
schema_params = set(i for i in data_idx.keys() if len(i.split(".")) == 2)
|
1603
|
-
# elements.append(
|
1604
|
-
# {
|
1605
|
-
# "iterations_idx": [self.num_elements + elem_idx],
|
1606
|
-
# "es_idx": self.num_element_sets - 1,
|
1607
|
-
# "seq_idx": ,
|
1608
|
-
# "src_idx": ,
|
1609
|
-
# }
|
1610
|
-
# )
|
1611
1758
|
elem_ID_i = self.workflow._store.add_element(
|
1612
1759
|
task_ID=self.insert_ID,
|
1613
1760
|
es_idx=self.num_element_sets - 1,
|
@@ -1622,27 +1769,7 @@ class WorkflowTask:
|
|
1622
1769
|
iter_IDs.append(iter_ID_i)
|
1623
1770
|
elem_IDs.append(elem_ID_i)
|
1624
1771
|
|
1625
|
-
# element_iterations.append(
|
1626
|
-
# {
|
1627
|
-
# "global_idx": self.workflow.num_element_iterations + elem_idx,
|
1628
|
-
# "data_idx": data_idx,
|
1629
|
-
# "EARs_initialised": False,
|
1630
|
-
# "actions": {},
|
1631
|
-
# "schema_parameters": list(schema_params),
|
1632
|
-
# "loop_idx": {},
|
1633
|
-
# }
|
1634
|
-
# )
|
1635
|
-
|
1636
|
-
# self.workflow._store.add_elements(
|
1637
|
-
# self.index,
|
1638
|
-
# self.insert_ID,
|
1639
|
-
# elements,
|
1640
|
-
# element_iterations,
|
1641
|
-
# )
|
1642
1772
|
self._pending_element_IDs += elem_IDs
|
1643
|
-
# self._pending_num_elements += len(element_data_idx)
|
1644
|
-
# self._pending_num_element_iterations += len(element_data_idx)
|
1645
|
-
|
1646
1773
|
self.initialise_EARs()
|
1647
1774
|
|
1648
1775
|
return iter_IDs
|
@@ -1895,9 +2022,13 @@ class WorkflowTask:
|
|
1895
2022
|
return
|
1896
2023
|
|
1897
2024
|
if path[0] == "inputs":
|
2025
|
+
try:
|
2026
|
+
path_1 = path[1].split("[")[0]
|
2027
|
+
except IndexError:
|
2028
|
+
path_1 = path[1]
|
1898
2029
|
for i in self.template.schemas:
|
1899
2030
|
for j in i.inputs:
|
1900
|
-
if j.parameter.typ ==
|
2031
|
+
if j.parameter.typ == path_1:
|
1901
2032
|
return j.parameter
|
1902
2033
|
|
1903
2034
|
elif path[0] == "outputs":
|
@@ -1950,29 +2081,33 @@ class WorkflowTask:
|
|
1950
2081
|
|
1951
2082
|
data = []
|
1952
2083
|
for data_idx_i_j in data_idx_i:
|
1953
|
-
|
1954
|
-
|
1955
|
-
|
1956
|
-
data_j = Path(self.workflow.path) / param_j.file["path"]
|
1957
|
-
else:
|
1958
|
-
data_j = Path(param_j.file["path"])
|
1959
|
-
data_j = data_j.as_posix()
|
2084
|
+
if path_i[0] == "repeats":
|
2085
|
+
# data is an integer repeats index, rather than a parameter ID:
|
2086
|
+
data_j = data_idx_i_j
|
1960
2087
|
else:
|
1961
|
-
|
1962
|
-
if
|
1963
|
-
|
1964
|
-
|
1965
|
-
# get the class method that should be invoked to initialise the
|
1966
|
-
# object:
|
1967
|
-
if is_multi:
|
1968
|
-
sources.append(param_j.source)
|
2088
|
+
param_j = self.workflow.get_parameter(data_idx_i_j)
|
2089
|
+
if param_j.file:
|
2090
|
+
if param_j.file["store_contents"]:
|
2091
|
+
data_j = Path(self.workflow.path) / param_j.file["path"]
|
1969
2092
|
else:
|
1970
|
-
|
1971
|
-
|
1972
|
-
|
1973
|
-
|
1974
|
-
|
1975
|
-
|
2093
|
+
data_j = Path(param_j.file["path"])
|
2094
|
+
data_j = data_j.as_posix()
|
2095
|
+
else:
|
2096
|
+
data_j = param_j.data
|
2097
|
+
if parameter and len(path_i) == 2:
|
2098
|
+
# retrieve the source if this is a non-sub parameter, so we can,
|
2099
|
+
# in the case that there is an associated `ParameterValue` class,
|
2100
|
+
# get the class method that should be invoked to initialise the
|
2101
|
+
# object:
|
2102
|
+
if is_multi:
|
2103
|
+
sources.append(param_j.source)
|
2104
|
+
else:
|
2105
|
+
sources = param_j.source
|
2106
|
+
if raise_on_unset and not param_j.is_set:
|
2107
|
+
raise UnsetParameterDataError(
|
2108
|
+
f"Element data path {path!r} resolves to unset data for (at "
|
2109
|
+
f"least) data index path: {path_i!r}."
|
2110
|
+
)
|
1976
2111
|
if not is_multi:
|
1977
2112
|
data = data_j
|
1978
2113
|
else:
|
@@ -2000,9 +2135,10 @@ class WorkflowTask:
|
|
2000
2135
|
current_value = default
|
2001
2136
|
|
2002
2137
|
if parameter and parameter._value_class:
|
2138
|
+
# TODO: retrieve value class method case-insensitively!
|
2003
2139
|
if isinstance(current_value, dict):
|
2004
2140
|
# return a ParameterValue instance:
|
2005
|
-
method_name = sources
|
2141
|
+
method_name = sources.get("value_class_method")
|
2006
2142
|
if method_name:
|
2007
2143
|
method = getattr(parameter._value_class, method_name)
|
2008
2144
|
else:
|
@@ -2013,7 +2149,7 @@ class WorkflowTask:
|
|
2013
2149
|
# return a list of ParameterValue instances:
|
2014
2150
|
current_value_ = []
|
2015
2151
|
for cur_val, src in zip(current_value, sources):
|
2016
|
-
method_name = src
|
2152
|
+
method_name = src.get("value_class_method")
|
2017
2153
|
if method_name:
|
2018
2154
|
method = getattr(parameter._value_class, method_name)
|
2019
2155
|
else:
|