hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a180__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/_version.py +1 -1
- hpcflow/data/demo_data_manifest/__init__.py +3 -0
- hpcflow/sdk/__init__.py +4 -1
- hpcflow/sdk/app.py +160 -15
- hpcflow/sdk/cli.py +14 -0
- hpcflow/sdk/cli_common.py +83 -0
- hpcflow/sdk/config/__init__.py +4 -0
- hpcflow/sdk/config/callbacks.py +25 -2
- hpcflow/sdk/config/cli.py +4 -1
- hpcflow/sdk/config/config.py +188 -14
- hpcflow/sdk/config/config_file.py +91 -3
- hpcflow/sdk/config/errors.py +33 -0
- hpcflow/sdk/core/__init__.py +2 -0
- hpcflow/sdk/core/actions.py +492 -35
- hpcflow/sdk/core/cache.py +22 -0
- hpcflow/sdk/core/command_files.py +221 -5
- hpcflow/sdk/core/commands.py +57 -0
- hpcflow/sdk/core/element.py +407 -8
- hpcflow/sdk/core/environment.py +92 -0
- hpcflow/sdk/core/errors.py +245 -61
- hpcflow/sdk/core/json_like.py +72 -14
- hpcflow/sdk/core/loop.py +122 -21
- hpcflow/sdk/core/loop_cache.py +34 -9
- hpcflow/sdk/core/object_list.py +172 -26
- hpcflow/sdk/core/parallel.py +14 -0
- hpcflow/sdk/core/parameters.py +478 -25
- hpcflow/sdk/core/rule.py +31 -1
- hpcflow/sdk/core/run_dir_files.py +12 -2
- hpcflow/sdk/core/task.py +407 -80
- hpcflow/sdk/core/task_schema.py +70 -9
- hpcflow/sdk/core/test_utils.py +35 -0
- hpcflow/sdk/core/utils.py +101 -4
- hpcflow/sdk/core/validation.py +13 -1
- hpcflow/sdk/core/workflow.py +316 -96
- hpcflow/sdk/core/zarr_io.py +23 -0
- hpcflow/sdk/data/__init__.py +13 -0
- hpcflow/sdk/demo/__init__.py +3 -0
- hpcflow/sdk/helper/__init__.py +3 -0
- hpcflow/sdk/helper/cli.py +9 -0
- hpcflow/sdk/helper/helper.py +28 -0
- hpcflow/sdk/helper/watcher.py +33 -0
- hpcflow/sdk/log.py +40 -0
- hpcflow/sdk/persistence/__init__.py +14 -4
- hpcflow/sdk/persistence/base.py +289 -23
- hpcflow/sdk/persistence/json.py +29 -0
- hpcflow/sdk/persistence/pending.py +217 -107
- hpcflow/sdk/persistence/store_resource.py +58 -2
- hpcflow/sdk/persistence/utils.py +8 -0
- hpcflow/sdk/persistence/zarr.py +68 -1
- hpcflow/sdk/runtime.py +52 -10
- hpcflow/sdk/submission/__init__.py +3 -0
- hpcflow/sdk/submission/jobscript.py +198 -9
- hpcflow/sdk/submission/jobscript_info.py +13 -0
- hpcflow/sdk/submission/schedulers/__init__.py +60 -0
- hpcflow/sdk/submission/schedulers/direct.py +53 -0
- hpcflow/sdk/submission/schedulers/sge.py +45 -7
- hpcflow/sdk/submission/schedulers/slurm.py +45 -8
- hpcflow/sdk/submission/schedulers/utils.py +4 -0
- hpcflow/sdk/submission/shells/__init__.py +11 -1
- hpcflow/sdk/submission/shells/base.py +32 -1
- hpcflow/sdk/submission/shells/bash.py +36 -1
- hpcflow/sdk/submission/shells/os_version.py +18 -6
- hpcflow/sdk/submission/shells/powershell.py +22 -0
- hpcflow/sdk/submission/submission.py +88 -3
- hpcflow/sdk/typing.py +10 -1
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/METADATA +1 -1
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/RECORD +70 -70
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/core/task.py
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
"""
|
2
|
+
Tasks are components of workflows.
|
3
|
+
"""
|
4
|
+
|
1
5
|
from __future__ import annotations
|
2
6
|
from collections import defaultdict
|
3
7
|
import copy
|
@@ -66,18 +70,59 @@ class InputStatus:
|
|
66
70
|
|
67
71
|
"""
|
68
72
|
|
73
|
+
#: True if a default value is available.
|
69
74
|
has_default: bool
|
75
|
+
#: True if the input is required by one or more actions. An input may not be required
|
76
|
+
#: if it is only used in the generation of inputs files, and those input files are
|
77
|
+
#: passed to the element set directly.
|
70
78
|
is_required: bool
|
79
|
+
#: True if the input is locally provided in the element set.
|
71
80
|
is_provided: bool
|
72
81
|
|
73
82
|
@property
|
74
83
|
def is_extra(self):
|
75
|
-
"""
|
84
|
+
"""True if the input is provided but not required."""
|
76
85
|
return self.is_provided and not self.is_required
|
77
86
|
|
78
87
|
|
79
88
|
class ElementSet(JSONLike):
|
80
|
-
"""Class to represent a
|
89
|
+
"""Class to represent a parameterisation of a new set of elements.
|
90
|
+
|
91
|
+
Parameters
|
92
|
+
----------
|
93
|
+
inputs: list[~hpcflow.app.InputValue]
|
94
|
+
Inputs to the set of elements.
|
95
|
+
input_files: list[~hpcflow.app.InputFile]
|
96
|
+
Input files to the set of elements.
|
97
|
+
sequences: list[~hpcflow.app.ValueSequence]
|
98
|
+
Input value sequences to parameterise over.
|
99
|
+
resources: ~hpcflow.app.ResourceList
|
100
|
+
Resources to use for the set of elements.
|
101
|
+
repeats: list[dict]
|
102
|
+
Description of how to repeat the set of elements.
|
103
|
+
groups: list[~hpcflow.app.ElementGroup]
|
104
|
+
Groupings in the set of elements.
|
105
|
+
input_sources: dict[str, ~hpcflow.app.InputSource]
|
106
|
+
Input source descriptors.
|
107
|
+
nesting_order: dict[str, int]
|
108
|
+
How to handle nesting of iterations.
|
109
|
+
env_preset: str
|
110
|
+
Which environment preset to use. Don't use at same time as ``environments``.
|
111
|
+
environments: dict
|
112
|
+
Environment descriptors to use. Don't use at same time as ``env_preset``.
|
113
|
+
sourceable_elem_iters: list[int]
|
114
|
+
If specified, a list of global element iteration indices from which inputs for
|
115
|
+
the new elements associated with this element set may be sourced. If not
|
116
|
+
specified, all workflow element iterations are considered sourceable.
|
117
|
+
allow_non_coincident_task_sources: bool
|
118
|
+
If True, if more than one parameter is sourced from the same task, then allow
|
119
|
+
these sources to come from distinct element sub-sets. If False (default),
|
120
|
+
only the intersection of element sub-sets for all parameters are included.
|
121
|
+
merge_envs: bool
|
122
|
+
If True, merge ``environments`` into ``resources`` using the "any" scope. If
|
123
|
+
False, ``environments`` are ignored. This is required on first initialisation,
|
124
|
+
but not on subsequent re-initialisation from a persistent workflow.
|
125
|
+
"""
|
81
126
|
|
82
127
|
_child_objects = (
|
83
128
|
ChildObjectSpec(
|
@@ -137,35 +182,34 @@ class ElementSet(JSONLike):
|
|
137
182
|
allow_non_coincident_task_sources: Optional[bool] = False,
|
138
183
|
merge_envs: Optional[bool] = True,
|
139
184
|
):
|
140
|
-
|
141
|
-
Parameters
|
142
|
-
----------
|
143
|
-
sourceable_elem_iters
|
144
|
-
If specified, a list of global element iteration indices from which inputs for
|
145
|
-
the new elements associated with this element set may be sourced. If not
|
146
|
-
specified, all workflow element iterations are considered sourceable.
|
147
|
-
allow_non_coincident_task_sources
|
148
|
-
If True, if more than one parameter is sourced from the same task, then allow
|
149
|
-
these sources to come from distinct element sub-sets. If False (default),
|
150
|
-
only the intersection of element sub-sets for all parameters are included.
|
151
|
-
merge_envs
|
152
|
-
If True, merge `environments` into `resources` using the "any" scope. If
|
153
|
-
False, `environments` are ignored. This is required on first initialisation,
|
154
|
-
but not on subsequent re-initialisation from a persistent workflow.
|
155
|
-
"""
|
156
|
-
|
185
|
+
#: Inputs to the set of elements.
|
157
186
|
self.inputs = inputs or []
|
187
|
+
#: Input files to the set of elements.
|
158
188
|
self.input_files = input_files or []
|
189
|
+
#: Description of how to repeat the set of elements.
|
159
190
|
self.repeats = repeats or []
|
191
|
+
#: Groupings in the set of elements.
|
160
192
|
self.groups = groups or []
|
193
|
+
#: Resources to use for the set of elements.
|
161
194
|
self.resources = self.app.ResourceList.normalise(resources)
|
195
|
+
#: Input value sequences to parameterise over.
|
162
196
|
self.sequences = sequences or []
|
197
|
+
#: Input source descriptors.
|
163
198
|
self.input_sources = input_sources or {}
|
199
|
+
#: How to handle nesting of iterations.
|
164
200
|
self.nesting_order = nesting_order or {}
|
201
|
+
#: Which environment preset to use.
|
165
202
|
self.env_preset = env_preset
|
203
|
+
#: Environment descriptors to use.
|
166
204
|
self.environments = environments
|
205
|
+
#: List of global element iteration indices from which inputs for
|
206
|
+
#: the new elements associated with this element set may be sourced.
|
207
|
+
#: If ``None``, all iterations are valid.
|
167
208
|
self.sourceable_elem_iters = sourceable_elem_iters
|
209
|
+
#: Whether to allow sources to come from distinct element sub-sets.
|
168
210
|
self.allow_non_coincident_task_sources = allow_non_coincident_task_sources
|
211
|
+
#: Whether to merge ``environments`` into ``resources`` using the "any" scope
|
212
|
+
#: on first initialisation.
|
169
213
|
self.merge_envs = merge_envs
|
170
214
|
|
171
215
|
self._validate()
|
@@ -235,6 +279,9 @@ class ElementSet(JSONLike):
|
|
235
279
|
|
236
280
|
@property
|
237
281
|
def task_template(self):
|
282
|
+
"""
|
283
|
+
The abstract task this was derived from.
|
284
|
+
"""
|
238
285
|
return self._task_template
|
239
286
|
|
240
287
|
@task_template.setter
|
@@ -244,11 +291,14 @@ class ElementSet(JSONLike):
|
|
244
291
|
|
245
292
|
@property
|
246
293
|
def input_types(self):
|
294
|
+
"""
|
295
|
+
The input types of the inputs to this element set.
|
296
|
+
"""
|
247
297
|
return [i.labelled_type for i in self.inputs]
|
248
298
|
|
249
299
|
@property
|
250
300
|
def element_local_idx_range(self):
|
251
|
-
"""
|
301
|
+
"""Indices of elements belonging to this element set."""
|
252
302
|
return tuple(self._element_local_idx_range)
|
253
303
|
|
254
304
|
def _validate(self):
|
@@ -376,6 +426,9 @@ class ElementSet(JSONLike):
|
|
376
426
|
element_sets=None,
|
377
427
|
sourceable_elem_iters=None,
|
378
428
|
):
|
429
|
+
"""
|
430
|
+
Make an instance after validating some argument combinations.
|
431
|
+
"""
|
379
432
|
args = (
|
380
433
|
inputs,
|
381
434
|
input_files,
|
@@ -417,18 +470,30 @@ class ElementSet(JSONLike):
|
|
417
470
|
|
418
471
|
@property
|
419
472
|
def defined_input_types(self):
|
473
|
+
"""
|
474
|
+
The input types to this element set.
|
475
|
+
"""
|
420
476
|
return self._defined_input_types
|
421
477
|
|
422
478
|
@property
|
423
479
|
def undefined_input_types(self):
|
480
|
+
"""
|
481
|
+
The input types to the abstract task that aren't related to this element set.
|
482
|
+
"""
|
424
483
|
return self.task_template.all_schema_input_types - self.defined_input_types
|
425
484
|
|
426
485
|
def get_sequence_from_path(self, sequence_path):
|
427
|
-
|
428
|
-
|
429
|
-
|
486
|
+
"""
|
487
|
+
Get the value sequence for the given path, if it exists.
|
488
|
+
"""
|
489
|
+
for seq in self.sequences:
|
490
|
+
if seq.path == sequence_path:
|
491
|
+
return seq
|
430
492
|
|
431
493
|
def get_defined_parameter_types(self):
|
494
|
+
"""
|
495
|
+
Get the parameter types of this element set.
|
496
|
+
"""
|
432
497
|
out = []
|
433
498
|
for inp in self.inputs:
|
434
499
|
if not inp.is_sub_value:
|
@@ -439,6 +504,9 @@ class ElementSet(JSONLike):
|
|
439
504
|
return out
|
440
505
|
|
441
506
|
def get_defined_sub_parameter_types(self):
|
507
|
+
"""
|
508
|
+
Get the sub-parameter types of this element set.
|
509
|
+
"""
|
442
510
|
out = []
|
443
511
|
for inp in self.inputs:
|
444
512
|
if inp.is_sub_value:
|
@@ -449,35 +517,48 @@ class ElementSet(JSONLike):
|
|
449
517
|
return out
|
450
518
|
|
451
519
|
def get_locally_defined_inputs(self):
|
520
|
+
"""
|
521
|
+
Get the input types that this element set defines.
|
522
|
+
"""
|
452
523
|
return self.get_defined_parameter_types() + self.get_defined_sub_parameter_types()
|
453
524
|
|
454
|
-
def get_sequence_by_path(self, path):
|
455
|
-
for seq in self.sequences:
|
456
|
-
if seq.path == path:
|
457
|
-
return seq
|
458
|
-
|
459
525
|
@property
|
460
526
|
def index(self):
|
527
|
+
"""
|
528
|
+
The index of this element set in its' template task's collection of sets.
|
529
|
+
"""
|
461
530
|
for idx, element_set in enumerate(self.task_template.element_sets):
|
462
531
|
if element_set is self:
|
463
532
|
return idx
|
464
533
|
|
465
534
|
@property
|
466
535
|
def task(self):
|
536
|
+
"""
|
537
|
+
The concrete task corresponding to this element set.
|
538
|
+
"""
|
467
539
|
return self.task_template.workflow_template.workflow.tasks[
|
468
540
|
self.task_template.index
|
469
541
|
]
|
470
542
|
|
471
543
|
@property
|
472
544
|
def elements(self):
|
545
|
+
"""
|
546
|
+
The elements in this element set.
|
547
|
+
"""
|
473
548
|
return self.task.elements[slice(*self.element_local_idx_range)]
|
474
549
|
|
475
550
|
@property
|
476
551
|
def element_iterations(self):
|
552
|
+
"""
|
553
|
+
The iterations in this element set.
|
554
|
+
"""
|
477
555
|
return [j for i in self.elements for j in i.iterations]
|
478
556
|
|
479
557
|
@property
|
480
558
|
def elem_iter_IDs(self):
|
559
|
+
"""
|
560
|
+
The IDs of the iterations in this element set.
|
561
|
+
"""
|
481
562
|
return [i.id_ for i in self.element_iterations]
|
482
563
|
|
483
564
|
def get_task_dependencies(self, as_objects=False):
|
@@ -510,8 +591,18 @@ class ElementSet(JSONLike):
|
|
510
591
|
|
511
592
|
|
512
593
|
class OutputLabel(JSONLike):
|
513
|
-
"""
|
514
|
-
outputs
|
594
|
+
"""
|
595
|
+
Schema input labels that should be applied to a subset of task outputs.
|
596
|
+
|
597
|
+
Parameters
|
598
|
+
----------
|
599
|
+
parameter:
|
600
|
+
Name of a parameter.
|
601
|
+
label:
|
602
|
+
Label to apply to the parameter.
|
603
|
+
where: ~hpcflow.app.ElementFilter
|
604
|
+
Optional filtering rule
|
605
|
+
"""
|
515
606
|
|
516
607
|
_child_objects = (
|
517
608
|
ChildObjectSpec(
|
@@ -526,22 +617,50 @@ class OutputLabel(JSONLike):
|
|
526
617
|
label: str,
|
527
618
|
where: Optional[List[app.ElementFilter]] = None,
|
528
619
|
) -> None:
|
620
|
+
#: Name of a parameter.
|
529
621
|
self.parameter = parameter
|
622
|
+
#: Label to apply to the parameter.
|
530
623
|
self.label = label
|
624
|
+
#: Filtering rule.
|
531
625
|
self.where = where
|
532
626
|
|
533
627
|
|
534
628
|
class Task(JSONLike):
|
535
|
-
"""
|
629
|
+
"""
|
630
|
+
Parametrisation of an isolated task for which a subset of input values are given
|
536
631
|
"locally". The remaining input values are expected to be satisfied by other
|
537
632
|
tasks/imports in the workflow.
|
538
633
|
|
539
634
|
Parameters
|
540
635
|
----------
|
541
|
-
schema
|
542
|
-
A `TaskSchema` object or a list of
|
543
|
-
|
636
|
+
schema: ~hpcflow.app.TaskSchema | list[~hpcflow.app.TaskSchema]
|
637
|
+
A (list of) `TaskSchema` object(s) and/or a (list of) strings that are task
|
638
|
+
schema names that uniquely identify a task schema. If strings are provided,
|
639
|
+
the `TaskSchema` object will be fetched from the known task schemas loaded by
|
640
|
+
the app configuration.
|
641
|
+
repeats: list[dict]
|
642
|
+
groups: list[~hpcflow.app.ElementGroup]
|
643
|
+
resources: dict
|
644
|
+
inputs: list[~hpcflow.app.InputValue]
|
544
645
|
A list of `InputValue` objects.
|
646
|
+
input_files: list[~hpcflow.app.InputFile]
|
647
|
+
sequences: list[~hpcflow.app.ValueSequence]
|
648
|
+
input_sources: dict[str, ~hpcflow.app.InputSource]
|
649
|
+
nesting_order: list
|
650
|
+
env_preset: str
|
651
|
+
environments: dict[str, dict]
|
652
|
+
allow_non_coincident_task_sources: bool
|
653
|
+
If True, if more than one parameter is sourced from the same task, then allow
|
654
|
+
these sources to come from distinct element sub-sets. If False (default),
|
655
|
+
only the intersection of element sub-sets for all parameters are included.
|
656
|
+
element_sets: list[ElementSet]
|
657
|
+
output_labels: list[OutputLabel]
|
658
|
+
sourceable_elem_iters: list[int]
|
659
|
+
merge_envs: bool
|
660
|
+
If True, merge environment presets (set via the element set `env_preset` key)
|
661
|
+
into `resources` using the "any" scope. If False, these presets are ignored.
|
662
|
+
This is required on first initialisation, but not on subsequent
|
663
|
+
re-initialisation from a persistent workflow.
|
545
664
|
"""
|
546
665
|
|
547
666
|
_child_objects = (
|
@@ -585,25 +704,6 @@ class Task(JSONLike):
|
|
585
704
|
sourceable_elem_iters: Optional[List[int]] = None,
|
586
705
|
merge_envs: Optional[bool] = True,
|
587
706
|
):
|
588
|
-
"""
|
589
|
-
Parameters
|
590
|
-
----------
|
591
|
-
schema
|
592
|
-
A (list of) `TaskSchema` object(s) and/or a (list of) strings that are task
|
593
|
-
schema names that uniquely identify a task schema. If strings are provided,
|
594
|
-
the `TaskSchema` object will be fetched from the known task schemas loaded by
|
595
|
-
the app configuration.
|
596
|
-
allow_non_coincident_task_sources
|
597
|
-
If True, if more than one parameter is sourced from the same task, then allow
|
598
|
-
these sources to come from distinct element sub-sets. If False (default),
|
599
|
-
only the intersection of element sub-sets for all parameters are included.
|
600
|
-
merge_envs
|
601
|
-
If True, merge environment presets (set via the element set `env_preset` key)
|
602
|
-
into `resources` using the "any" scope. If False, these presets are ignored.
|
603
|
-
This is required on first initialisation, but not on subsequent
|
604
|
-
re-initialisation from a persistent workflow.
|
605
|
-
"""
|
606
|
-
|
607
707
|
# TODO: allow init via specifying objective and/or method and/or implementation
|
608
708
|
# (lists of) strs e.g.: Task(
|
609
709
|
# objective='simulate_VE_loading',
|
@@ -652,6 +752,8 @@ class Task(JSONLike):
|
|
652
752
|
sourceable_elem_iters=sourceable_elem_iters,
|
653
753
|
)
|
654
754
|
self._output_labels = output_labels or []
|
755
|
+
#: Whether to merge ``environments`` into ``resources`` using the "any" scope
|
756
|
+
#: on first initialisation.
|
655
757
|
self.merge_envs = merge_envs
|
656
758
|
|
657
759
|
# appended to when new element sets are added and reset on dump to disk:
|
@@ -660,6 +762,7 @@ class Task(JSONLike):
|
|
660
762
|
self._validate()
|
661
763
|
self._name = self._get_name()
|
662
764
|
|
765
|
+
#: The template workflow that this task is within.
|
663
766
|
self.workflow_template = None # assigned by parent WorkflowTemplate
|
664
767
|
self._insert_ID = None
|
665
768
|
self._dir_name = None
|
@@ -800,6 +903,9 @@ class Task(JSONLike):
|
|
800
903
|
}
|
801
904
|
|
802
905
|
def set_sequence_parameters(self, element_set):
|
906
|
+
"""
|
907
|
+
Set up parameters parsed by value sequences.
|
908
|
+
"""
|
803
909
|
# set ValueSequence Parameter objects:
|
804
910
|
for seq in element_set.sequences:
|
805
911
|
if seq.input_type:
|
@@ -882,6 +988,11 @@ class Task(JSONLike):
|
|
882
988
|
return output_data_indices
|
883
989
|
|
884
990
|
def prepare_element_resolution(self, element_set, input_data_indices):
|
991
|
+
"""
|
992
|
+
Set up the resolution of details of elements
|
993
|
+
(especially multiplicities and how iterations are nested)
|
994
|
+
within an element set.
|
995
|
+
"""
|
885
996
|
multiplicities = []
|
886
997
|
for path_i, inp_idx_i in input_data_indices.items():
|
887
998
|
multiplicities.append(
|
@@ -917,6 +1028,9 @@ class Task(JSONLike):
|
|
917
1028
|
|
918
1029
|
@property
|
919
1030
|
def index(self):
|
1031
|
+
"""
|
1032
|
+
The index of this task within the workflow's tasks.
|
1033
|
+
"""
|
920
1034
|
if self.workflow_template:
|
921
1035
|
return self.workflow_template.tasks.index(self)
|
922
1036
|
else:
|
@@ -924,6 +1038,9 @@ class Task(JSONLike):
|
|
924
1038
|
|
925
1039
|
@property
|
926
1040
|
def output_labels(self):
|
1041
|
+
"""
|
1042
|
+
The labels on the outputs of the task.
|
1043
|
+
"""
|
927
1044
|
return self._output_labels
|
928
1045
|
|
929
1046
|
@property
|
@@ -1113,11 +1230,14 @@ class Task(JSONLike):
|
|
1113
1230
|
|
1114
1231
|
@property
|
1115
1232
|
def schemas(self) -> List[app.TaskSchema]:
|
1233
|
+
"""
|
1234
|
+
All the task schemas.
|
1235
|
+
"""
|
1116
1236
|
return self._schemas
|
1117
1237
|
|
1118
1238
|
@property
|
1119
1239
|
def schema(self) -> app.TaskSchema:
|
1120
|
-
"""
|
1240
|
+
"""The single task schema, if only one, else raises."""
|
1121
1241
|
if len(self._schemas) == 1:
|
1122
1242
|
return self._schemas[0]
|
1123
1243
|
else:
|
@@ -1128,52 +1248,81 @@ class Task(JSONLike):
|
|
1128
1248
|
|
1129
1249
|
@property
|
1130
1250
|
def element_sets(self):
|
1251
|
+
"""
|
1252
|
+
The element sets.
|
1253
|
+
"""
|
1131
1254
|
return self._element_sets + self._pending_element_sets
|
1132
1255
|
|
1133
1256
|
@property
|
1134
1257
|
def num_element_sets(self):
|
1258
|
+
"""
|
1259
|
+
The number of element sets.
|
1260
|
+
"""
|
1135
1261
|
return len(self.element_sets)
|
1136
1262
|
|
1137
1263
|
@property
|
1138
1264
|
def insert_ID(self):
|
1265
|
+
"""
|
1266
|
+
Insertion ID.
|
1267
|
+
"""
|
1139
1268
|
return self._insert_ID
|
1140
1269
|
|
1141
1270
|
@property
|
1142
1271
|
def dir_name(self):
|
1143
|
-
"
|
1272
|
+
"""
|
1273
|
+
Artefact directory name.
|
1274
|
+
"""
|
1144
1275
|
return self._dir_name
|
1145
1276
|
|
1146
1277
|
@property
|
1147
1278
|
def name(self):
|
1279
|
+
"""
|
1280
|
+
Task name.
|
1281
|
+
"""
|
1148
1282
|
return self._name
|
1149
1283
|
|
1150
1284
|
@property
|
1151
1285
|
def objective(self):
|
1286
|
+
"""
|
1287
|
+
The goal of this task.
|
1288
|
+
"""
|
1152
1289
|
return self.schemas[0].objective
|
1153
1290
|
|
1154
1291
|
@property
|
1155
1292
|
def all_schema_inputs(self) -> Tuple[app.SchemaInput]:
|
1293
|
+
"""
|
1294
|
+
The inputs to this task's schemas.
|
1295
|
+
"""
|
1156
1296
|
return tuple(inp_j for schema_i in self.schemas for inp_j in schema_i.inputs)
|
1157
1297
|
|
1158
1298
|
@property
|
1159
1299
|
def all_schema_outputs(self) -> Tuple[app.SchemaOutput]:
|
1300
|
+
"""
|
1301
|
+
The outputs from this task's schemas.
|
1302
|
+
"""
|
1160
1303
|
return tuple(inp_j for schema_i in self.schemas for inp_j in schema_i.outputs)
|
1161
1304
|
|
1162
1305
|
@property
|
1163
1306
|
def all_schema_input_types(self):
|
1164
|
-
"""
|
1307
|
+
"""The set of all schema input types (over all specified schemas)."""
|
1165
1308
|
return {inp_j for schema_i in self.schemas for inp_j in schema_i.input_types}
|
1166
1309
|
|
1167
1310
|
@property
|
1168
1311
|
def all_schema_input_normalised_paths(self):
|
1312
|
+
"""
|
1313
|
+
Normalised paths for all schema input types.
|
1314
|
+
"""
|
1169
1315
|
return {f"inputs.{i}" for i in self.all_schema_input_types}
|
1170
1316
|
|
1171
1317
|
@property
|
1172
1318
|
def all_schema_output_types(self):
|
1173
|
-
"""
|
1319
|
+
"""The set of all schema output types (over all specified schemas)."""
|
1174
1320
|
return {out_j for schema_i in self.schemas for out_j in schema_i.output_types}
|
1175
1321
|
|
1176
1322
|
def get_schema_action(self, idx):
|
1323
|
+
"""
|
1324
|
+
Get the schema action at the given index.
|
1325
|
+
"""
|
1177
1326
|
_idx = 0
|
1178
1327
|
for schema in self.schemas:
|
1179
1328
|
for action in schema.actions:
|
@@ -1183,6 +1332,9 @@ class Task(JSONLike):
|
|
1183
1332
|
raise ValueError(f"No action in task {self.name!r} with index {idx!r}.")
|
1184
1333
|
|
1185
1334
|
def all_schema_actions(self) -> Iterator[Tuple[int, app.Action]]:
|
1335
|
+
"""
|
1336
|
+
Get all the schema actions and their indices.
|
1337
|
+
"""
|
1186
1338
|
idx = 0
|
1187
1339
|
for schema in self.schemas:
|
1188
1340
|
for action in schema.actions:
|
@@ -1191,6 +1343,9 @@ class Task(JSONLike):
|
|
1191
1343
|
|
1192
1344
|
@property
|
1193
1345
|
def num_all_schema_actions(self) -> int:
|
1346
|
+
"""
|
1347
|
+
The total number of schema actions.
|
1348
|
+
"""
|
1194
1349
|
num = 0
|
1195
1350
|
for schema in self.schemas:
|
1196
1351
|
for _ in schema.actions:
|
@@ -1199,6 +1354,9 @@ class Task(JSONLike):
|
|
1199
1354
|
|
1200
1355
|
@property
|
1201
1356
|
def all_sourced_normalised_paths(self):
|
1357
|
+
"""
|
1358
|
+
All the sourced normalised paths, including of sub-values.
|
1359
|
+
"""
|
1202
1360
|
sourced_input_types = []
|
1203
1361
|
for elem_set in self.element_sets:
|
1204
1362
|
for inp in elem_set.inputs:
|
@@ -1280,14 +1438,23 @@ class Task(JSONLike):
|
|
1280
1438
|
|
1281
1439
|
@property
|
1282
1440
|
def defined_input_types(self):
|
1441
|
+
"""
|
1442
|
+
The input types defined by this task.
|
1443
|
+
"""
|
1283
1444
|
return self._defined_input_types
|
1284
1445
|
|
1285
1446
|
@property
|
1286
1447
|
def undefined_input_types(self):
|
1448
|
+
"""
|
1449
|
+
The schema's input types that this task doesn't define.
|
1450
|
+
"""
|
1287
1451
|
return self.all_schema_input_types - self.defined_input_types
|
1288
1452
|
|
1289
1453
|
@property
|
1290
1454
|
def undefined_inputs(self):
|
1455
|
+
"""
|
1456
|
+
The task's inputs that are undefined.
|
1457
|
+
"""
|
1291
1458
|
return [
|
1292
1459
|
inp_j
|
1293
1460
|
for schema_i in self.schemas
|
@@ -1323,6 +1490,9 @@ class Task(JSONLike):
|
|
1323
1490
|
def add_group(
|
1324
1491
|
self, name: str, where: app.ElementFilter, group_by_distinct: app.ParameterPath
|
1325
1492
|
):
|
1493
|
+
"""
|
1494
|
+
Add an element group to this task.
|
1495
|
+
"""
|
1326
1496
|
group = ElementGroup(name=name, where=where, group_by_distinct=group_by_distinct)
|
1327
1497
|
self.groups.add_object(group)
|
1328
1498
|
|
@@ -1346,7 +1516,20 @@ class Task(JSONLike):
|
|
1346
1516
|
|
1347
1517
|
|
1348
1518
|
class WorkflowTask:
|
1349
|
-
"""
|
1519
|
+
"""
|
1520
|
+
Represents a :py:class:`Task` that is bound to a :py:class:`Workflow`.
|
1521
|
+
|
1522
|
+
Parameters
|
1523
|
+
----------
|
1524
|
+
workflow:
|
1525
|
+
The workflow that the task is bound to.
|
1526
|
+
template:
|
1527
|
+
The task template that this binds.
|
1528
|
+
index:
|
1529
|
+
Where in the workflow's list of tasks is this one.
|
1530
|
+
element_IDs:
|
1531
|
+
The IDs of the elements of this task.
|
1532
|
+
"""
|
1350
1533
|
|
1351
1534
|
_app_attr = "app"
|
1352
1535
|
|
@@ -1379,6 +1562,18 @@ class WorkflowTask:
|
|
1379
1562
|
|
1380
1563
|
@classmethod
|
1381
1564
|
def new_empty_task(cls, workflow: app.Workflow, template: app.Task, index: int):
|
1565
|
+
"""
|
1566
|
+
Make a new instance without any elements set up yet.
|
1567
|
+
|
1568
|
+
Parameters
|
1569
|
+
----------
|
1570
|
+
workflow:
|
1571
|
+
The workflow that the task is bound to.
|
1572
|
+
template:
|
1573
|
+
The task template that this binds.
|
1574
|
+
index:
|
1575
|
+
Where in the workflow's list of tasks is this one.
|
1576
|
+
"""
|
1382
1577
|
obj = cls(
|
1383
1578
|
workflow=workflow,
|
1384
1579
|
template=template,
|
@@ -1389,61 +1584,103 @@ class WorkflowTask:
|
|
1389
1584
|
|
1390
1585
|
@property
|
1391
1586
|
def workflow(self):
|
1587
|
+
"""
|
1588
|
+
The workflow this task is bound to.
|
1589
|
+
"""
|
1392
1590
|
return self._workflow
|
1393
1591
|
|
1394
1592
|
@property
|
1395
1593
|
def template(self):
|
1594
|
+
"""
|
1595
|
+
The template for this task.
|
1596
|
+
"""
|
1396
1597
|
return self._template
|
1397
1598
|
|
1398
1599
|
@property
|
1399
1600
|
def index(self):
|
1601
|
+
"""
|
1602
|
+
The index of this task within its workflow.
|
1603
|
+
"""
|
1400
1604
|
return self._index
|
1401
1605
|
|
1402
1606
|
@property
|
1403
1607
|
def element_IDs(self):
|
1608
|
+
"""
|
1609
|
+
The IDs of elements associated with this task.
|
1610
|
+
"""
|
1404
1611
|
return self._element_IDs + self._pending_element_IDs
|
1405
1612
|
|
1406
1613
|
@property
|
1407
1614
|
def num_elements(self):
|
1615
|
+
"""
|
1616
|
+
The number of elements associated with this task.
|
1617
|
+
"""
|
1408
1618
|
return len(self.element_IDs)
|
1409
1619
|
|
1410
1620
|
@property
|
1411
1621
|
def num_actions(self):
|
1622
|
+
"""
|
1623
|
+
The number of actions in this task.
|
1624
|
+
"""
|
1412
1625
|
return self.template.num_all_schema_actions
|
1413
1626
|
|
1414
1627
|
@property
|
1415
1628
|
def name(self):
|
1629
|
+
"""
|
1630
|
+
The name of this task based on its template.
|
1631
|
+
"""
|
1416
1632
|
return self.template.name
|
1417
1633
|
|
1418
1634
|
@property
|
1419
1635
|
def unique_name(self):
|
1636
|
+
"""
|
1637
|
+
The unique name for this task specifically.
|
1638
|
+
"""
|
1420
1639
|
return self.workflow.get_task_unique_names()[self.index]
|
1421
1640
|
|
1422
1641
|
@property
|
1423
1642
|
def insert_ID(self):
|
1643
|
+
"""
|
1644
|
+
The insertion ID of the template task.
|
1645
|
+
"""
|
1424
1646
|
return self.template.insert_ID
|
1425
1647
|
|
1426
1648
|
@property
|
1427
1649
|
def dir_name(self):
|
1650
|
+
"""
|
1651
|
+
The name of the directory for the task's temporary files.
|
1652
|
+
"""
|
1428
1653
|
return self.template.dir_name
|
1429
1654
|
|
1430
1655
|
@property
|
1431
1656
|
def num_element_sets(self):
|
1657
|
+
"""
|
1658
|
+
The number of element sets associated with this task.
|
1659
|
+
"""
|
1432
1660
|
return self.template.num_element_sets
|
1433
1661
|
|
1434
1662
|
@property
|
1435
1663
|
@TimeIt.decorator
|
1436
1664
|
def elements(self):
|
1665
|
+
"""
|
1666
|
+
The elements associated with this task.
|
1667
|
+
"""
|
1437
1668
|
if self._elements is None:
|
1438
1669
|
self._elements = self.app.Elements(self)
|
1439
1670
|
return self._elements
|
1440
1671
|
|
1441
1672
|
def get_dir_name(self, loop_idx: Dict[str, int] = None) -> str:
|
1673
|
+
"""
|
1674
|
+
Get the directory name for a particular iteration.
|
1675
|
+
"""
|
1442
1676
|
if not loop_idx:
|
1443
1677
|
return self.dir_name
|
1444
1678
|
return self.dir_name + "_" + "_".join((f"{k}-{v}" for k, v in loop_idx.items()))
|
1445
1679
|
|
1446
1680
|
def get_all_element_iterations(self) -> Dict[int, app.ElementIteration]:
|
1681
|
+
"""
|
1682
|
+
Get the iterations known by the task's elements.
|
1683
|
+
"""
|
1447
1684
|
return {j.id_: j for i in self.elements for j in i.iterations}
|
1448
1685
|
|
1449
1686
|
def _make_new_elements_persistent(
|
@@ -1633,7 +1870,7 @@ class WorkflowTask:
|
|
1633
1870
|
source_idx[key] = [inp_src_idx] * len(grp_idx)
|
1634
1871
|
if key in sequence_idx:
|
1635
1872
|
sequence_idx.pop(key)
|
1636
|
-
seq = element_set.
|
1873
|
+
seq = element_set.get_sequence_from_path(key)
|
1637
1874
|
|
1638
1875
|
elif inp_src.source_type is InputSourceType.DEFAULT:
|
1639
1876
|
grp_idx = [def_val._value_group_idx]
|
@@ -1950,6 +2187,9 @@ class WorkflowTask:
|
|
1950
2187
|
sequence_indices,
|
1951
2188
|
source_indices,
|
1952
2189
|
):
|
2190
|
+
"""
|
2191
|
+
Create information about new elements in this task.
|
2192
|
+
"""
|
1953
2193
|
new_elements = []
|
1954
2194
|
element_sequence_indices = {}
|
1955
2195
|
element_src_indices = {}
|
@@ -1984,12 +2224,12 @@ class WorkflowTask:
|
|
1984
2224
|
|
1985
2225
|
@property
|
1986
2226
|
def upstream_tasks(self):
|
1987
|
-
"""
|
2227
|
+
"""All workflow tasks that are upstream from this task."""
|
1988
2228
|
return [task for task in self.workflow.tasks[: self.index]]
|
1989
2229
|
|
1990
2230
|
@property
|
1991
2231
|
def downstream_tasks(self):
|
1992
|
-
"""
|
2232
|
+
"""All workflow tasks that are downstream from this task."""
|
1993
2233
|
return [task for task in self.workflow.tasks[self.index + 1 :]]
|
1994
2234
|
|
1995
2235
|
@staticmethod
|
@@ -2246,6 +2486,22 @@ class WorkflowTask:
|
|
2246
2486
|
propagate_to=None,
|
2247
2487
|
return_indices=False,
|
2248
2488
|
):
|
2489
|
+
"""
|
2490
|
+
Add elements to this task.
|
2491
|
+
|
2492
|
+
Parameters
|
2493
|
+
----------
|
2494
|
+
sourceable_elem_iters : list of int, optional
|
2495
|
+
If specified, a list of global element iteration indices from which inputs
|
2496
|
+
may be sourced. If not specified, all workflow element iterations are
|
2497
|
+
considered sourceable.
|
2498
|
+
propagate_to : dict[str, ElementPropagation]
|
2499
|
+
Propagate the new elements downstream to the specified tasks.
|
2500
|
+
return_indices : bool
|
2501
|
+
If True, return the list of indices of the newly added elements. False by
|
2502
|
+
default.
|
2503
|
+
|
2504
|
+
"""
|
2249
2505
|
propagate_to = self.app.ElementPropagation._prepare_propagate_to_dict(
|
2250
2506
|
propagate_to, self.workflow
|
2251
2507
|
)
|
@@ -2281,21 +2537,7 @@ class WorkflowTask:
|
|
2281
2537
|
propagate_to: Dict[str, app.ElementPropagation] = None,
|
2282
2538
|
return_indices: bool = False,
|
2283
2539
|
):
|
2284
|
-
"""Add more elements to this task.
|
2285
|
-
|
2286
|
-
Parameters
|
2287
|
-
----------
|
2288
|
-
sourceable_elem_iters : list of int, optional
|
2289
|
-
If specified, a list of global element iteration indices from which inputs
|
2290
|
-
may be sourced. If not specified, all workflow element iterations are
|
2291
|
-
considered sourceable.
|
2292
|
-
propagate_to : dict of [str, ElementPropagation]
|
2293
|
-
Propagate the new elements downstream to the specified tasks.
|
2294
|
-
return_indices : bool, optional
|
2295
|
-
If True, return the list of indices of the newly added elements. False by
|
2296
|
-
default.
|
2297
|
-
|
2298
|
-
"""
|
2540
|
+
"""Add more elements to this task."""
|
2299
2541
|
|
2300
2542
|
if base_element is not None:
|
2301
2543
|
if base_element.task is not self:
|
@@ -2467,13 +2709,22 @@ class WorkflowTask:
|
|
2467
2709
|
|
2468
2710
|
@property
|
2469
2711
|
def inputs(self):
|
2712
|
+
"""
|
2713
|
+
Inputs to this task.
|
2714
|
+
"""
|
2470
2715
|
return self.app.TaskInputParameters(self)
|
2471
2716
|
|
2472
2717
|
@property
|
2473
2718
|
def outputs(self):
|
2719
|
+
"""
|
2720
|
+
Outputs from this task.
|
2721
|
+
"""
|
2474
2722
|
return self.app.TaskOutputParameters(self)
|
2475
2723
|
|
2476
2724
|
def get(self, path, raise_on_missing=False, default=None):
|
2725
|
+
"""
|
2726
|
+
Get a parameter known to this task by its path.
|
2727
|
+
"""
|
2477
2728
|
return self.app.Parameters(
|
2478
2729
|
self,
|
2479
2730
|
path=path,
|
@@ -2866,6 +3117,15 @@ class WorkflowTask:
|
|
2866
3117
|
|
2867
3118
|
|
2868
3119
|
class Elements:
|
3120
|
+
"""
|
3121
|
+
The elements of a task. Iterable.
|
3122
|
+
|
3123
|
+
Parameters
|
3124
|
+
----------
|
3125
|
+
task:
|
3126
|
+
The task this will be the elements of.
|
3127
|
+
"""
|
3128
|
+
|
2869
3129
|
__slots__ = ("_task",)
|
2870
3130
|
|
2871
3131
|
def __init__(self, task: app.WorkflowTask):
|
@@ -2881,6 +3141,9 @@ class Elements:
|
|
2881
3141
|
|
2882
3142
|
@property
|
2883
3143
|
def task(self):
|
3144
|
+
"""
|
3145
|
+
The task this is the elements of.
|
3146
|
+
"""
|
2884
3147
|
return self._task
|
2885
3148
|
|
2886
3149
|
@TimeIt.decorator
|
@@ -2924,13 +3187,38 @@ class Elements:
|
|
2924
3187
|
|
2925
3188
|
@dataclass
|
2926
3189
|
class Parameters:
|
3190
|
+
"""
|
3191
|
+
The parameters of a (workflow-bound) task. Iterable.
|
3192
|
+
|
3193
|
+
Parameters
|
3194
|
+
----------
|
3195
|
+
task: WorkflowTask
|
3196
|
+
The task these are the parameters of.
|
3197
|
+
path: str
|
3198
|
+
The path to the parameter or parameters.
|
3199
|
+
return_element_parameters: bool
|
3200
|
+
Whether to return element parameters.
|
3201
|
+
raise_on_missing: bool
|
3202
|
+
Whether to raise an exception on a missing parameter.
|
3203
|
+
raise_on_unset: bool
|
3204
|
+
Whether to raise an exception on an unset parameter.
|
3205
|
+
default:
|
3206
|
+
A default value to use when the parameter is absent.
|
3207
|
+
"""
|
3208
|
+
|
2927
3209
|
_app_attr = "_app"
|
2928
3210
|
|
3211
|
+
#: The task these are the parameters of.
|
2929
3212
|
task: app.WorkflowTask
|
3213
|
+
#: The path to the parameter or parameters.
|
2930
3214
|
path: str
|
3215
|
+
#: Whether to return element parameters.
|
2931
3216
|
return_element_parameters: bool
|
3217
|
+
#: Whether to raise an exception on a missing parameter.
|
2932
3218
|
raise_on_missing: Optional[bool] = False
|
3219
|
+
#: Whether to raise an exception on an unset parameter.
|
2933
3220
|
raise_on_unset: Optional[bool] = False
|
3221
|
+
#: A default value to use when the parameter is absent.
|
2934
3222
|
default: Optional[Any] = None
|
2935
3223
|
|
2936
3224
|
@TimeIt.decorator
|
@@ -2990,10 +3278,19 @@ class Parameters:
|
|
2990
3278
|
|
2991
3279
|
@dataclass
|
2992
3280
|
class TaskInputParameters:
|
2993
|
-
"""
|
3281
|
+
"""
|
3282
|
+
For retrieving schema input parameters across all elements.
|
3283
|
+
Treat as an unmodifiable namespace.
|
3284
|
+
|
3285
|
+
Parameters
|
3286
|
+
----------
|
3287
|
+
task:
|
3288
|
+
The task that this represents the input parameters of.
|
3289
|
+
"""
|
2994
3290
|
|
2995
3291
|
_app_attr = "_app"
|
2996
3292
|
|
3293
|
+
#: The task that this represents the input parameters of.
|
2997
3294
|
task: app.WorkflowTask
|
2998
3295
|
|
2999
3296
|
def __getattr__(self, name):
|
@@ -3016,10 +3313,19 @@ class TaskInputParameters:
|
|
3016
3313
|
|
3017
3314
|
@dataclass
|
3018
3315
|
class TaskOutputParameters:
|
3019
|
-
"""
|
3316
|
+
"""
|
3317
|
+
For retrieving schema output parameters across all elements.
|
3318
|
+
Treat as an unmodifiable namespace.
|
3319
|
+
|
3320
|
+
Parameters
|
3321
|
+
----------
|
3322
|
+
task:
|
3323
|
+
The task that this represents the output parameters of.
|
3324
|
+
"""
|
3020
3325
|
|
3021
3326
|
_app_attr = "_app"
|
3022
3327
|
|
3328
|
+
#: The task that this represents the output parameters of.
|
3023
3329
|
task: app.WorkflowTask
|
3024
3330
|
|
3025
3331
|
def __getattr__(self, name):
|
@@ -3042,17 +3348,38 @@ class TaskOutputParameters:
|
|
3042
3348
|
|
3043
3349
|
@dataclass
|
3044
3350
|
class ElementPropagation:
|
3045
|
-
"""
|
3046
|
-
|
3351
|
+
"""
|
3352
|
+
Class to represent how a newly added element set should propagate to a given
|
3353
|
+
downstream task.
|
3354
|
+
|
3355
|
+
Parameters
|
3356
|
+
----------
|
3357
|
+
task:
|
3358
|
+
The task this is propagating to.
|
3359
|
+
nesting_order:
|
3360
|
+
The nesting order information.
|
3361
|
+
input_sources:
|
3362
|
+
The input source information.
|
3363
|
+
"""
|
3047
3364
|
|
3048
3365
|
_app_attr = "app"
|
3049
3366
|
|
3367
|
+
#: The task this is propagating to.
|
3050
3368
|
task: app.Task
|
3369
|
+
#: The nesting order information.
|
3051
3370
|
nesting_order: Optional[Dict] = None
|
3371
|
+
#: The input source information.
|
3052
3372
|
input_sources: Optional[Dict] = None
|
3053
3373
|
|
3054
3374
|
@property
|
3055
3375
|
def element_set(self):
|
3376
|
+
"""
|
3377
|
+
The element set that this propagates from.
|
3378
|
+
|
3379
|
+
Note
|
3380
|
+
----
|
3381
|
+
Temporary property. May be moved or reinterpreted.
|
3382
|
+
"""
|
3056
3383
|
# TEMP property; for now just use the first element set as the base:
|
3057
3384
|
return self.task.template.element_sets[0]
|
3058
3385
|
|