hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a180__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/_version.py +1 -1
- hpcflow/data/demo_data_manifest/__init__.py +3 -0
- hpcflow/sdk/__init__.py +4 -1
- hpcflow/sdk/app.py +160 -15
- hpcflow/sdk/cli.py +14 -0
- hpcflow/sdk/cli_common.py +83 -0
- hpcflow/sdk/config/__init__.py +4 -0
- hpcflow/sdk/config/callbacks.py +25 -2
- hpcflow/sdk/config/cli.py +4 -1
- hpcflow/sdk/config/config.py +188 -14
- hpcflow/sdk/config/config_file.py +91 -3
- hpcflow/sdk/config/errors.py +33 -0
- hpcflow/sdk/core/__init__.py +2 -0
- hpcflow/sdk/core/actions.py +492 -35
- hpcflow/sdk/core/cache.py +22 -0
- hpcflow/sdk/core/command_files.py +221 -5
- hpcflow/sdk/core/commands.py +57 -0
- hpcflow/sdk/core/element.py +407 -8
- hpcflow/sdk/core/environment.py +92 -0
- hpcflow/sdk/core/errors.py +245 -61
- hpcflow/sdk/core/json_like.py +72 -14
- hpcflow/sdk/core/loop.py +122 -21
- hpcflow/sdk/core/loop_cache.py +34 -9
- hpcflow/sdk/core/object_list.py +172 -26
- hpcflow/sdk/core/parallel.py +14 -0
- hpcflow/sdk/core/parameters.py +478 -25
- hpcflow/sdk/core/rule.py +31 -1
- hpcflow/sdk/core/run_dir_files.py +12 -2
- hpcflow/sdk/core/task.py +407 -80
- hpcflow/sdk/core/task_schema.py +70 -9
- hpcflow/sdk/core/test_utils.py +35 -0
- hpcflow/sdk/core/utils.py +101 -4
- hpcflow/sdk/core/validation.py +13 -1
- hpcflow/sdk/core/workflow.py +316 -96
- hpcflow/sdk/core/zarr_io.py +23 -0
- hpcflow/sdk/data/__init__.py +13 -0
- hpcflow/sdk/demo/__init__.py +3 -0
- hpcflow/sdk/helper/__init__.py +3 -0
- hpcflow/sdk/helper/cli.py +9 -0
- hpcflow/sdk/helper/helper.py +28 -0
- hpcflow/sdk/helper/watcher.py +33 -0
- hpcflow/sdk/log.py +40 -0
- hpcflow/sdk/persistence/__init__.py +14 -4
- hpcflow/sdk/persistence/base.py +289 -23
- hpcflow/sdk/persistence/json.py +29 -0
- hpcflow/sdk/persistence/pending.py +217 -107
- hpcflow/sdk/persistence/store_resource.py +58 -2
- hpcflow/sdk/persistence/utils.py +8 -0
- hpcflow/sdk/persistence/zarr.py +68 -1
- hpcflow/sdk/runtime.py +52 -10
- hpcflow/sdk/submission/__init__.py +3 -0
- hpcflow/sdk/submission/jobscript.py +198 -9
- hpcflow/sdk/submission/jobscript_info.py +13 -0
- hpcflow/sdk/submission/schedulers/__init__.py +60 -0
- hpcflow/sdk/submission/schedulers/direct.py +53 -0
- hpcflow/sdk/submission/schedulers/sge.py +45 -7
- hpcflow/sdk/submission/schedulers/slurm.py +45 -8
- hpcflow/sdk/submission/schedulers/utils.py +4 -0
- hpcflow/sdk/submission/shells/__init__.py +11 -1
- hpcflow/sdk/submission/shells/base.py +32 -1
- hpcflow/sdk/submission/shells/bash.py +36 -1
- hpcflow/sdk/submission/shells/os_version.py +18 -6
- hpcflow/sdk/submission/shells/powershell.py +22 -0
- hpcflow/sdk/submission/submission.py +88 -3
- hpcflow/sdk/typing.py +10 -1
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/METADATA +1 -1
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/RECORD +70 -70
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/core/parameters.py
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
"""
|
2
|
+
Parameters represent information passed around within a workflow.
|
3
|
+
"""
|
4
|
+
|
1
5
|
from __future__ import annotations
|
2
6
|
import copy
|
3
7
|
from dataclasses import dataclass, field
|
@@ -49,40 +53,74 @@ def _process_demo_data_strings(app, value):
|
|
49
53
|
|
50
54
|
|
51
55
|
class ParameterValue:
|
56
|
+
"""
|
57
|
+
The value handler for a parameter.
|
58
|
+
|
59
|
+
Intended to be subclassed.
|
60
|
+
"""
|
61
|
+
|
52
62
|
_typ = None
|
53
63
|
_sub_parameters = {}
|
54
64
|
|
55
65
|
def to_dict(self):
|
66
|
+
"""
|
67
|
+
Serialise this parameter value as a dictionary.
|
68
|
+
"""
|
56
69
|
if hasattr(self, "__dict__"):
|
57
70
|
return dict(self.__dict__)
|
58
71
|
elif hasattr(self, "__slots__"):
|
59
72
|
return {k: getattr(self, k) for k in self.__slots__}
|
60
73
|
|
61
74
|
def prepare_JSON_dump(self) -> Dict:
|
75
|
+
"""
|
76
|
+
Prepare this parameter value for serialisation as JSON.
|
77
|
+
"""
|
62
78
|
raise NotImplementedError
|
63
79
|
|
64
80
|
def dump_to_HDF5_group(self, group):
|
81
|
+
"""
|
82
|
+
Write this parameter value to an HDF5 group.
|
83
|
+
"""
|
65
84
|
raise NotImplementedError
|
66
85
|
|
67
86
|
@classmethod
|
68
87
|
def save_from_HDF5_group(cls, group, param_id: int, workflow):
|
88
|
+
"""
|
89
|
+
Extract a parameter value from an HDF5 group.
|
90
|
+
"""
|
69
91
|
raise NotImplementedError
|
70
92
|
|
71
93
|
@classmethod
|
72
94
|
def save_from_JSON(cls, data, param_id: int, workflow):
|
95
|
+
"""
|
96
|
+
Extract a parameter value from JSON data.
|
97
|
+
"""
|
73
98
|
raise NotImplementedError
|
74
99
|
|
75
100
|
|
76
101
|
class ParameterPropagationMode(enum.Enum):
|
102
|
+
"""
|
103
|
+
How a parameter is propagated.
|
104
|
+
"""
|
105
|
+
|
106
|
+
#: Parameter is propagated implicitly.
|
77
107
|
IMPLICIT = 0
|
108
|
+
#: Parameter is propagated explicitly.
|
78
109
|
EXPLICIT = 1
|
110
|
+
#: Parameter is never propagated.
|
79
111
|
NEVER = 2
|
80
112
|
|
81
113
|
|
82
114
|
@dataclass
|
83
115
|
class ParameterPath(JSONLike):
|
116
|
+
"""
|
117
|
+
Path to a parameter.
|
118
|
+
"""
|
119
|
+
|
84
120
|
# TODO: unused?
|
121
|
+
#: The path to the parameter.
|
85
122
|
path: Sequence[Union[str, int, float]]
|
123
|
+
#: The task in which to look up the parameter.
|
86
124
|
task: Optional[
|
87
125
|
Union[app.TaskTemplate, app.TaskSchema]
|
88
126
|
] = None # default is "current" task
|
@@ -90,6 +128,28 @@ class ParameterPath(JSONLike):
|
|
90
128
|
|
91
129
|
@dataclass
|
92
130
|
class Parameter(JSONLike):
|
131
|
+
"""
|
132
|
+
A general parameter to a workflow task.
|
133
|
+
|
134
|
+
Parameters
|
135
|
+
----------
|
136
|
+
typ:
|
137
|
+
Type code.
|
138
|
+
Used to look up the :py:class:`ParameterValue` for this parameter,
|
139
|
+
if any.
|
140
|
+
is_file:
|
141
|
+
Whether this parameter represents a file.
|
142
|
+
sub_parameters: list[SubParameter]
|
143
|
+
Any parameters packed within this one.
|
144
|
+
_value_class: type[ParameterValue]
|
145
|
+
Class that provides the implementation of this parameter's values.
|
146
|
+
Not normally directly user-managed.
|
147
|
+
_hash_value:
|
148
|
+
Hash of this class. Not normally user-managed.
|
149
|
+
_validation:
|
150
|
+
Validation schema.
|
151
|
+
"""
|
152
|
+
|
93
153
|
_validation_schema = "parameters_spec_schema.yaml"
|
94
154
|
_child_objects = (
|
95
155
|
ChildObjectSpec(
|
@@ -102,8 +162,12 @@ class Parameter(JSONLike):
|
|
102
162
|
),
|
103
163
|
)
|
104
164
|
|
165
|
+
#: Type code. Used to look up the :py:class:`ParameterValue` for this parameter,
|
166
|
+
#: if any.
|
105
167
|
typ: str
|
168
|
+
#: Whether this parameter represents a file.
|
106
169
|
is_file: bool = False
|
170
|
+
#: Any parameters packed within this one.
|
107
171
|
sub_parameters: List[app.SubParameter] = field(default_factory=lambda: [])
|
108
172
|
_value_class: Any = None
|
109
173
|
_hash_value: Optional[str] = field(default=None, repr=False)
|
@@ -158,17 +222,35 @@ class Parameter(JSONLike):
|
|
158
222
|
|
159
223
|
@property
|
160
224
|
def url_slug(self) -> str:
|
225
|
+
"""
|
226
|
+
Representation of this parameter as part of a URL.
|
227
|
+
"""
|
161
228
|
return self.typ.lower().replace("_", "-")
|
162
229
|
|
163
230
|
|
164
231
|
@dataclass
|
165
232
|
class SubParameter:
|
233
|
+
"""
|
234
|
+
A parameter that is a component of another parameter.
|
235
|
+
"""
|
236
|
+
|
237
|
+
#: How to find this within the containing paraneter.
|
166
238
|
address: Address
|
239
|
+
#: The containing main parameter.
|
167
240
|
parameter: app.Parameter
|
168
241
|
|
169
242
|
|
170
243
|
@dataclass
|
171
244
|
class SchemaParameter(JSONLike):
|
245
|
+
"""
|
246
|
+
A parameter bound in a schema.
|
247
|
+
|
248
|
+
Parameters
|
249
|
+
----------
|
250
|
+
parameter: Parameter
|
251
|
+
The parameter.
|
252
|
+
"""
|
253
|
+
|
172
254
|
_app_attr = "app"
|
173
255
|
|
174
256
|
_child_objects = (
|
@@ -189,14 +271,26 @@ class SchemaParameter(JSONLike):
|
|
189
271
|
|
190
272
|
@property
|
191
273
|
def name(self):
|
274
|
+
"""
|
275
|
+
The name of the parameter.
|
276
|
+
"""
|
192
277
|
return self.parameter.name
|
193
278
|
|
194
279
|
@property
|
195
280
|
def typ(self):
|
281
|
+
"""
|
282
|
+
The type code of the parameter.
|
283
|
+
"""
|
196
284
|
return self.parameter.typ
|
197
285
|
|
198
286
|
|
199
287
|
class NullDefault(enum.Enum):
|
288
|
+
"""
|
289
|
+
Sentinel value used to distinguish an explicit null.
|
290
|
+
"""
|
291
|
+
|
292
|
+
#: Special sentinel.
|
293
|
+
#: Used in situations where otherwise a JSON object or array would be.
|
200
294
|
NULL = 0
|
201
295
|
|
202
296
|
|
@@ -206,13 +300,13 @@ class SchemaInput(SchemaParameter):
|
|
206
300
|
|
207
301
|
Parameters
|
208
302
|
----------
|
209
|
-
parameter
|
303
|
+
parameter:
|
210
304
|
The parameter (i.e. type) of this schema input.
|
211
|
-
multiple
|
305
|
+
multiple:
|
212
306
|
If True, expect one or more of these parameters defined in the workflow,
|
213
307
|
distinguished by a string label in square brackets. For example `p1[0]` for a
|
214
308
|
parameter `p1`.
|
215
|
-
labels
|
309
|
+
labels:
|
216
310
|
Dict whose keys represent the string labels that distinguish multiple parameters
|
217
311
|
if `multiple` is `True`. Use the key "*" to mean all labels not matching
|
218
312
|
other label keys. If `multiple` is `False`, this will default to a
|
@@ -220,10 +314,10 @@ class SchemaInput(SchemaParameter):
|
|
220
314
|
`True`, this will default to a single-item dict with the catch-all key:
|
221
315
|
`{{"*": {{}}}}`. On initialisation, remaining keyword-arguments are treated as default
|
222
316
|
values for the dict values of `labels`.
|
223
|
-
default_value
|
317
|
+
default_value:
|
224
318
|
The default value for this input parameter. This is itself a default value that
|
225
319
|
will be applied to all `labels` values if a "default_value" key does not exist.
|
226
|
-
propagation_mode
|
320
|
+
propagation_mode:
|
227
321
|
Determines how this input should propagate through the workflow. This is a default
|
228
322
|
value that will be applied to all `labels` values if a "propagation_mode" key does
|
229
323
|
not exist. By default, the input is allowed to be used in downstream tasks simply
|
@@ -232,7 +326,7 @@ class SchemaInput(SchemaParameter):
|
|
232
326
|
the downstream task `input_sources` for it to be used, and "never", meaning that
|
233
327
|
the parameter must not be used in downstream tasks and will be inaccessible to
|
234
328
|
those tasks.
|
235
|
-
group
|
329
|
+
group:
|
236
330
|
Determines the name of the element group from which this input should be sourced.
|
237
331
|
This is a default value that will be applied to all `labels` if a "group" key
|
238
332
|
does not exist.
|
@@ -270,8 +364,12 @@ class SchemaInput(SchemaParameter):
|
|
270
364
|
except ValueError:
|
271
365
|
parameter = self.app.Parameter(parameter)
|
272
366
|
|
367
|
+
#: The parameter (i.e. type) of this schema input.
|
273
368
|
self.parameter = parameter
|
369
|
+
#: Whether to expect more than of these parameters defined in the workflow.
|
274
370
|
self.multiple = multiple
|
371
|
+
#: Dict whose keys represent the string labels that distinguish multiple
|
372
|
+
#: parameters if `multiple` is `True`.
|
275
373
|
self.labels = labels
|
276
374
|
|
277
375
|
if self.labels is None:
|
@@ -395,6 +493,9 @@ class SchemaInput(SchemaParameter):
|
|
395
493
|
|
396
494
|
@property
|
397
495
|
def default_value(self):
|
496
|
+
"""
|
497
|
+
The default value of the input.
|
498
|
+
"""
|
398
499
|
if not self.multiple:
|
399
500
|
if "default_value" in self.single_labelled_data:
|
400
501
|
return self.single_labelled_data["default_value"]
|
@@ -403,28 +504,46 @@ class SchemaInput(SchemaParameter):
|
|
403
504
|
|
404
505
|
@property
|
405
506
|
def task_schema(self):
|
507
|
+
"""
|
508
|
+
The schema containing this input.
|
509
|
+
"""
|
406
510
|
return self._task_schema
|
407
511
|
|
408
512
|
@property
|
409
513
|
def all_labelled_types(self):
|
514
|
+
"""
|
515
|
+
The types of the input labels.
|
516
|
+
"""
|
410
517
|
return list(f"{self.typ}{f'[{i}]' if i else ''}" for i in self.labels)
|
411
518
|
|
412
519
|
@property
|
413
520
|
def single_label(self):
|
521
|
+
"""
|
522
|
+
The label of this input, assuming it is not mulitple.
|
523
|
+
"""
|
414
524
|
if not self.multiple:
|
415
525
|
return next(iter(self.labels))
|
416
526
|
|
417
527
|
@property
|
418
528
|
def single_labelled_type(self):
|
529
|
+
"""
|
530
|
+
The type code of this input, assuming it is not mulitple.
|
531
|
+
"""
|
419
532
|
if not self.multiple:
|
420
533
|
return next(iter(self.labelled_info()))["labelled_type"]
|
421
534
|
|
422
535
|
@property
|
423
536
|
def single_labelled_data(self):
|
537
|
+
"""
|
538
|
+
The value of this input, assuming it is not mulitple.
|
539
|
+
"""
|
424
540
|
if not self.multiple:
|
425
541
|
return self.labels[self.single_label]
|
426
542
|
|
427
543
|
def labelled_info(self):
|
544
|
+
"""
|
545
|
+
Get descriptors for all the labels associated with this input.
|
546
|
+
"""
|
428
547
|
for k, v in self.labels.items():
|
429
548
|
label = f"[{k}]" if k else ""
|
430
549
|
dct = {
|
@@ -458,6 +577,9 @@ class SchemaInput(SchemaParameter):
|
|
458
577
|
|
459
578
|
@property
|
460
579
|
def input_or_output(self):
|
580
|
+
"""
|
581
|
+
Whether this is an input or output. Always ``input``.
|
582
|
+
"""
|
461
583
|
return "input"
|
462
584
|
|
463
585
|
|
@@ -465,11 +587,16 @@ class SchemaInput(SchemaParameter):
|
|
465
587
|
class SchemaOutput(SchemaParameter):
|
466
588
|
"""A Parameter as outputted from particular task."""
|
467
589
|
|
590
|
+
#: The basic parameter this supplies.
|
468
591
|
parameter: Parameter
|
592
|
+
#: How this output propagates.
|
469
593
|
propagation_mode: ParameterPropagationMode = ParameterPropagationMode.IMPLICIT
|
470
594
|
|
471
595
|
@property
|
472
596
|
def input_or_output(self):
|
597
|
+
"""
|
598
|
+
Whether this is an input or output. Always ``output``.
|
599
|
+
"""
|
473
600
|
return "output"
|
474
601
|
|
475
602
|
def __repr__(self) -> str:
|
@@ -483,6 +610,11 @@ class SchemaOutput(SchemaParameter):
|
|
483
610
|
|
484
611
|
@dataclass
|
485
612
|
class BuiltinSchemaParameter:
|
613
|
+
"""
|
614
|
+
A parameter of a built-in schema.
|
615
|
+
"""
|
616
|
+
|
617
|
+
# TODO: Is this used anywhere?
|
486
618
|
# builtin inputs (resources,parameter_perturbations,method,implementation
|
487
619
|
# builtin outputs (time, memory use, node/hostname etc)
|
488
620
|
# - builtin parameters do not propagate to other tasks (since all tasks define the same
|
@@ -493,6 +625,23 @@ class BuiltinSchemaParameter:
|
|
493
625
|
|
494
626
|
|
495
627
|
class ValueSequence(JSONLike):
|
628
|
+
"""
|
629
|
+
A sequence of values.
|
630
|
+
|
631
|
+
Parameters
|
632
|
+
----------
|
633
|
+
path:
|
634
|
+
The path to this sequence.
|
635
|
+
values:
|
636
|
+
The values in this sequence.
|
637
|
+
nesting_order: int
|
638
|
+
A nesting order for this sequence. Can be used to compose sequences together.
|
639
|
+
label: str
|
640
|
+
A label for this sequence.
|
641
|
+
value_class_method: str
|
642
|
+
Name of a method used to generate sequence values. Not normally used directly.
|
643
|
+
"""
|
644
|
+
|
496
645
|
def __init__(
|
497
646
|
self,
|
498
647
|
path: str,
|
@@ -504,9 +653,13 @@ class ValueSequence(JSONLike):
|
|
504
653
|
label = str(label) if label is not None else ""
|
505
654
|
path, label = self._validate_parameter_path(path, label)
|
506
655
|
|
656
|
+
#: The path to this sequence.
|
507
657
|
self.path = path
|
658
|
+
#: The label of this sequence.
|
508
659
|
self.label = label
|
660
|
+
#: The nesting order for this sequence.
|
509
661
|
self.nesting_order = nesting_order
|
662
|
+
#: Name of a method used to generate sequence values.
|
510
663
|
self.value_class_method = value_class_method
|
511
664
|
|
512
665
|
if values is not None:
|
@@ -603,30 +756,48 @@ class ValueSequence(JSONLike):
|
|
603
756
|
|
604
757
|
@property
|
605
758
|
def parameter(self):
|
759
|
+
"""
|
760
|
+
The parameter this sequence supplies.
|
761
|
+
"""
|
606
762
|
return self._parameter
|
607
763
|
|
608
764
|
@property
|
609
765
|
def path_split(self):
|
766
|
+
"""
|
767
|
+
The components of ths path.
|
768
|
+
"""
|
610
769
|
if self._path_split is None:
|
611
770
|
self._path_split = self.path.split(".")
|
612
771
|
return self._path_split
|
613
772
|
|
614
773
|
@property
|
615
774
|
def path_type(self):
|
775
|
+
"""
|
776
|
+
The type of path this is.
|
777
|
+
"""
|
616
778
|
return self.path_split[0]
|
617
779
|
|
618
780
|
@property
|
619
781
|
def input_type(self):
|
782
|
+
"""
|
783
|
+
The type of input sequence this is, if it is one.
|
784
|
+
"""
|
620
785
|
if self.path_type == "inputs":
|
621
786
|
return self.path_split[1].replace(self._label_fmt, "")
|
622
787
|
|
623
788
|
@property
|
624
789
|
def input_path(self):
|
790
|
+
"""
|
791
|
+
The path of the input sequence this is, if it is one.
|
792
|
+
"""
|
625
793
|
if self.path_type == "inputs":
|
626
794
|
return ".".join(self.path_split[2:])
|
627
795
|
|
628
796
|
@property
|
629
797
|
def resource_scope(self):
|
798
|
+
"""
|
799
|
+
The scope of the resources this is, if it is one.
|
800
|
+
"""
|
630
801
|
if self.path_type == "resources":
|
631
802
|
return self.path_split[1]
|
632
803
|
|
@@ -641,6 +812,9 @@ class ValueSequence(JSONLike):
|
|
641
812
|
|
642
813
|
@property
|
643
814
|
def labelled_type(self):
|
815
|
+
"""
|
816
|
+
The labelled type of input sequence this is, if it is one.
|
817
|
+
"""
|
644
818
|
if self.input_type:
|
645
819
|
return f"{self.input_type}{self._label_fmt}"
|
646
820
|
|
@@ -750,12 +924,17 @@ class ValueSequence(JSONLike):
|
|
750
924
|
|
751
925
|
@property
|
752
926
|
def normalised_path(self):
|
927
|
+
"""
|
928
|
+
The path to this sequence.
|
929
|
+
"""
|
753
930
|
return self.path
|
754
931
|
|
755
932
|
@property
|
756
933
|
def normalised_inputs_path(self):
|
757
|
-
"""
|
758
|
-
inputs
|
934
|
+
"""
|
935
|
+
The normalised path without the "inputs" prefix, if the sequence is an
|
936
|
+
inputs sequence, else return None.
|
937
|
+
"""
|
759
938
|
|
760
939
|
if self.input_type:
|
761
940
|
if self.input_path:
|
@@ -802,6 +981,9 @@ class ValueSequence(JSONLike):
|
|
802
981
|
|
803
982
|
@property
|
804
983
|
def workflow(self):
|
984
|
+
"""
|
985
|
+
The workflow containing this sequence.
|
986
|
+
"""
|
805
987
|
if self._workflow:
|
806
988
|
return self._workflow
|
807
989
|
elif self._element_set:
|
@@ -809,6 +991,9 @@ class ValueSequence(JSONLike):
|
|
809
991
|
|
810
992
|
@property
|
811
993
|
def values(self):
|
994
|
+
"""
|
995
|
+
The values in this sequence.
|
996
|
+
"""
|
812
997
|
if self._values_group_idx is not None:
|
813
998
|
vals = []
|
814
999
|
for idx, pg_idx_i in enumerate(self._values_group_idx):
|
@@ -885,6 +1070,9 @@ class ValueSequence(JSONLike):
|
|
885
1070
|
label=None,
|
886
1071
|
**kwargs,
|
887
1072
|
):
|
1073
|
+
"""
|
1074
|
+
Build a sequence from a NumPy linear space.
|
1075
|
+
"""
|
888
1076
|
# TODO: save persistently as an array?
|
889
1077
|
args = {"start": start, "stop": stop, "num": num, **kwargs}
|
890
1078
|
values = cls._values_from_linear_space(**args)
|
@@ -905,6 +1093,9 @@ class ValueSequence(JSONLike):
|
|
905
1093
|
label=None,
|
906
1094
|
**kwargs,
|
907
1095
|
):
|
1096
|
+
"""
|
1097
|
+
Build a sequence from a NumPy geometric space.
|
1098
|
+
"""
|
908
1099
|
args = {"start": start, "stop": stop, "num": num, "endpoint": endpoint, **kwargs}
|
909
1100
|
values = cls._values_from_geometric_space(**args)
|
910
1101
|
obj = cls(values=values, path=path, nesting_order=nesting_order, label=label)
|
@@ -925,6 +1116,9 @@ class ValueSequence(JSONLike):
|
|
925
1116
|
label=None,
|
926
1117
|
**kwargs,
|
927
1118
|
):
|
1119
|
+
"""
|
1120
|
+
Build a sequence from a NumPy logarithmic space.
|
1121
|
+
"""
|
928
1122
|
args = {
|
929
1123
|
"start": start,
|
930
1124
|
"stop": stop,
|
@@ -950,6 +1144,9 @@ class ValueSequence(JSONLike):
|
|
950
1144
|
label=None,
|
951
1145
|
**kwargs,
|
952
1146
|
):
|
1147
|
+
"""
|
1148
|
+
Build a sequence from a range.
|
1149
|
+
"""
|
953
1150
|
# TODO: save persistently as an array?
|
954
1151
|
args = {"start": start, "stop": stop, "step": step, **kwargs}
|
955
1152
|
if isinstance(step, int):
|
@@ -982,6 +1179,9 @@ class ValueSequence(JSONLike):
|
|
982
1179
|
label=None,
|
983
1180
|
**kwargs,
|
984
1181
|
):
|
1182
|
+
"""
|
1183
|
+
Build a sequence from a simple file.
|
1184
|
+
"""
|
985
1185
|
args = {"file_path": file_path, **kwargs}
|
986
1186
|
values = cls._values_from_file(**args)
|
987
1187
|
obj = cls(
|
@@ -1009,6 +1209,8 @@ class ValueSequence(JSONLike):
|
|
1009
1209
|
**kwargs,
|
1010
1210
|
):
|
1011
1211
|
"""
|
1212
|
+
Build a sequence to cover a rectangle.
|
1213
|
+
|
1012
1214
|
Parameters
|
1013
1215
|
----------
|
1014
1216
|
coord:
|
@@ -1044,6 +1246,9 @@ class ValueSequence(JSONLike):
|
|
1044
1246
|
label=None,
|
1045
1247
|
**kwargs,
|
1046
1248
|
):
|
1249
|
+
"""
|
1250
|
+
Build a sequence from a uniform random number generator.
|
1251
|
+
"""
|
1047
1252
|
args = {"low": low, "high": high, "num": num, "seed": seed, **kwargs}
|
1048
1253
|
values = cls._values_from_random_uniform(**args)
|
1049
1254
|
obj = cls(values=values, path=path, nesting_order=nesting_order, label=label)
|
@@ -1111,6 +1316,9 @@ class AbstractInputValue(JSONLike):
|
|
1111
1316
|
|
1112
1317
|
@property
|
1113
1318
|
def workflow(self):
|
1319
|
+
"""
|
1320
|
+
The workflow containing this input value.
|
1321
|
+
"""
|
1114
1322
|
if self._workflow:
|
1115
1323
|
return self._workflow
|
1116
1324
|
elif self._element_set:
|
@@ -1120,6 +1328,9 @@ class AbstractInputValue(JSONLike):
|
|
1120
1328
|
|
1121
1329
|
@property
|
1122
1330
|
def value(self):
|
1331
|
+
"""
|
1332
|
+
The value itself.
|
1333
|
+
"""
|
1123
1334
|
if self._value_group_idx is not None:
|
1124
1335
|
val = self.workflow.get_parameter_data(self._value_group_idx)
|
1125
1336
|
if self._value_is_obj and self.parameter._value_class:
|
@@ -1132,31 +1343,44 @@ class AbstractInputValue(JSONLike):
|
|
1132
1343
|
|
1133
1344
|
@dataclass
|
1134
1345
|
class ValuePerturbation(AbstractInputValue):
|
1346
|
+
"""
|
1347
|
+
A perturbation applied to a value.
|
1348
|
+
"""
|
1349
|
+
|
1350
|
+
#: The name of this perturbation.
|
1135
1351
|
name: str
|
1352
|
+
#: The path to the value(s) to perturb.
|
1136
1353
|
path: Optional[Sequence[Union[str, int, float]]] = None
|
1354
|
+
#: The multiplicative factor to apply.
|
1137
1355
|
multiplicative_factor: Optional[Numeric] = 1
|
1356
|
+
#: The additive factor to apply.
|
1138
1357
|
additive_factor: Optional[Numeric] = 0
|
1139
1358
|
|
1140
1359
|
@classmethod
|
1141
1360
|
def from_spec(cls, spec):
|
1361
|
+
"""
|
1362
|
+
Construct an instance from a specification dictionary.
|
1363
|
+
"""
|
1142
1364
|
return cls(**spec)
|
1143
1365
|
|
1144
1366
|
|
1145
1367
|
class InputValue(AbstractInputValue):
|
1146
1368
|
"""
|
1369
|
+
An input value to a task.
|
1370
|
+
|
1147
1371
|
Parameters
|
1148
1372
|
----------
|
1149
|
-
parameter
|
1150
|
-
Parameter whose value is to be specified
|
1151
|
-
label
|
1373
|
+
parameter: Parameter | SchemaInput | str
|
1374
|
+
Parameter whose value is to be specified.
|
1375
|
+
label: str
|
1152
1376
|
Optional identifier to be used where the associated `SchemaInput` accepts multiple
|
1153
1377
|
parameters of the specified type. This will be cast to a string.
|
1154
|
-
value
|
1378
|
+
value: Any
|
1155
1379
|
The input parameter value.
|
1156
|
-
value_class_method
|
1380
|
+
value_class_method: How to obtain the real value.
|
1157
1381
|
A class method that can be invoked with the `value` attribute as keyword
|
1158
1382
|
arguments.
|
1159
|
-
path
|
1383
|
+
path: str
|
1160
1384
|
Dot-delimited path within the parameter's nested data structure for which `value`
|
1161
1385
|
should be set.
|
1162
1386
|
|
@@ -1188,9 +1412,16 @@ class InputValue(AbstractInputValue):
|
|
1188
1412
|
elif isinstance(parameter, SchemaInput):
|
1189
1413
|
parameter = parameter.parameter
|
1190
1414
|
|
1415
|
+
#: Parameter whose value is to be specified.
|
1191
1416
|
self.parameter = parameter
|
1417
|
+
#: Identifier to be used where the associated `SchemaInput` accepts multiple
|
1418
|
+
#: parameters of the specified type.
|
1192
1419
|
self.label = str(label) if label is not None else ""
|
1420
|
+
#: Dot-delimited path within the parameter's nested data structure for which
|
1421
|
+
#: `value` should be set.
|
1193
1422
|
self.path = (path.strip(".") if path else None) or None
|
1423
|
+
#: A class method that can be invoked with the `value` attribute as keyword
|
1424
|
+
#: arguments.
|
1194
1425
|
self.value_class_method = value_class_method
|
1195
1426
|
self._value = _process_demo_data_strings(self.app, value)
|
1196
1427
|
|
@@ -1293,15 +1524,24 @@ class InputValue(AbstractInputValue):
|
|
1293
1524
|
|
1294
1525
|
@property
|
1295
1526
|
def labelled_type(self):
|
1527
|
+
"""
|
1528
|
+
The labelled type of this input value.
|
1529
|
+
"""
|
1296
1530
|
label = f"[{self.label}]" if self.label else ""
|
1297
1531
|
return f"{self.parameter.typ}{label}"
|
1298
1532
|
|
1299
1533
|
@property
|
1300
1534
|
def normalised_inputs_path(self):
|
1535
|
+
"""
|
1536
|
+
The normalised input path without the ``inputs.`` prefix.
|
1537
|
+
"""
|
1301
1538
|
return f"{self.labelled_type}{f'.{self.path}' if self.path else ''}"
|
1302
1539
|
|
1303
1540
|
@property
|
1304
1541
|
def normalised_path(self):
|
1542
|
+
"""
|
1543
|
+
The full normalised input path.
|
1544
|
+
"""
|
1305
1545
|
return f"inputs.{self.normalised_inputs_path}"
|
1306
1546
|
|
1307
1547
|
def make_persistent(self, workflow: Any, source: Dict) -> Tuple[str, List[int], bool]:
|
@@ -1347,8 +1587,55 @@ class ResourceSpec(JSONLike):
|
|
1347
1587
|
`os_name` is used for retrieving a default shell name and for retrieving the correct
|
1348
1588
|
`Shell` class; when using WSL, it should still be `nt` (i.e. Windows).
|
1349
1589
|
|
1590
|
+
Parameters
|
1591
|
+
----------
|
1592
|
+
scope:
|
1593
|
+
Which scope does this apply to.
|
1594
|
+
scratch: str
|
1595
|
+
Which scratch space to use.
|
1596
|
+
parallel_mode: ParallelMode
|
1597
|
+
Which parallel mode to use.
|
1598
|
+
num_cores: int
|
1599
|
+
How many cores to request.
|
1600
|
+
num_cores_per_node: int
|
1601
|
+
How many cores per compute node to request.
|
1602
|
+
num_threads: int
|
1603
|
+
How many threads to request.
|
1604
|
+
num_nodes: int
|
1605
|
+
How many compute nodes to request.
|
1606
|
+
scheduler: str
|
1607
|
+
Which scheduler to use.
|
1608
|
+
shell: str
|
1609
|
+
Which system shell to use.
|
1610
|
+
use_job_array: bool
|
1611
|
+
Whether to use array jobs.
|
1612
|
+
max_array_items: int
|
1613
|
+
If using array jobs, up to how many items should be in the job array.
|
1614
|
+
time_limit: str
|
1615
|
+
How long to run for.
|
1616
|
+
scheduler_args: dict[str, Any]
|
1617
|
+
Additional arguments to pass to the scheduler.
|
1618
|
+
shell_args: dict[str, Any]
|
1619
|
+
Additional arguments to pass to the shell.
|
1620
|
+
os_name: str
|
1621
|
+
Which OS to use.
|
1622
|
+
environments: dict
|
1623
|
+
Which execution environments to use.
|
1624
|
+
SGE_parallel_env: str
|
1625
|
+
Which SGE parallel environment to request.
|
1626
|
+
SLURM_partition: str
|
1627
|
+
Which SLURM partition to request.
|
1628
|
+
SLURM_num_tasks: str
|
1629
|
+
How many SLURM tasks to request.
|
1630
|
+
SLURM_num_tasks_per_node: str
|
1631
|
+
How many SLURM tasks per compute node to request.
|
1632
|
+
SLURM_num_nodes: str
|
1633
|
+
How many compute nodes to request.
|
1634
|
+
SLURM_num_cpus_per_task: str
|
1635
|
+
How many CPU cores to ask for per SLURM task.
|
1350
1636
|
"""
|
1351
1637
|
|
1638
|
+
#: The names of parameters that may be used when making an instance of this class.
|
1352
1639
|
ALLOWED_PARAMETERS = {
|
1353
1640
|
"scratch",
|
1354
1641
|
"parallel_mode",
|
@@ -1407,6 +1694,7 @@ class ResourceSpec(JSONLike):
|
|
1407
1694
|
SLURM_num_nodes: Optional[str] = None,
|
1408
1695
|
SLURM_num_cpus_per_task: Optional[str] = None,
|
1409
1696
|
):
|
1697
|
+
#: Which scope does this apply to.
|
1410
1698
|
self.scope = scope or self.app.ActionScope.any()
|
1411
1699
|
if not isinstance(self.scope, self.app.ActionScope):
|
1412
1700
|
self.scope = self.app.ActionScope.from_json_like(self.scope)
|
@@ -1498,10 +1786,16 @@ class ResourceSpec(JSONLike):
|
|
1498
1786
|
|
1499
1787
|
@property
|
1500
1788
|
def normalised_resources_path(self):
|
1789
|
+
"""
|
1790
|
+
Standard name of this resource spec.
|
1791
|
+
"""
|
1501
1792
|
return self.scope.to_string()
|
1502
1793
|
|
1503
1794
|
@property
|
1504
1795
|
def normalised_path(self):
|
1796
|
+
"""
|
1797
|
+
Full name of this resource spec.
|
1798
|
+
"""
|
1505
1799
|
return f"resources.{self.normalised_resources_path}"
|
1506
1800
|
|
1507
1801
|
def to_dict(self):
|
@@ -1596,31 +1890,55 @@ class ResourceSpec(JSONLike):
|
|
1596
1890
|
|
1597
1891
|
@property
|
1598
1892
|
def scratch(self):
|
1599
|
-
|
1893
|
+
"""
|
1894
|
+
Which scratch space to use.
|
1895
|
+
|
1896
|
+
Todo
|
1897
|
+
----
|
1898
|
+
Currently unused, except in tests.
|
1899
|
+
"""
|
1600
1900
|
return self._get_value("scratch")
|
1601
1901
|
|
1602
1902
|
@property
|
1603
1903
|
def parallel_mode(self):
|
1904
|
+
"""
|
1905
|
+
Which parallel mode to use.
|
1906
|
+
"""
|
1604
1907
|
return self._get_value("parallel_mode")
|
1605
1908
|
|
1606
1909
|
@property
|
1607
1910
|
def num_cores(self):
|
1911
|
+
"""
|
1912
|
+
How many cores to request.
|
1913
|
+
"""
|
1608
1914
|
return self._get_value("num_cores")
|
1609
1915
|
|
1610
1916
|
@property
|
1611
1917
|
def num_cores_per_node(self):
|
1918
|
+
"""
|
1919
|
+
How many cores per compute node to request.
|
1920
|
+
"""
|
1612
1921
|
return self._get_value("num_cores_per_node")
|
1613
1922
|
|
1614
1923
|
@property
|
1615
1924
|
def num_nodes(self):
|
1925
|
+
"""
|
1926
|
+
How many compute nodes to request.
|
1927
|
+
"""
|
1616
1928
|
return self._get_value("num_nodes")
|
1617
1929
|
|
1618
1930
|
@property
|
1619
1931
|
def num_threads(self):
|
1932
|
+
"""
|
1933
|
+
How many threads to request.
|
1934
|
+
"""
|
1620
1935
|
return self._get_value("num_threads")
|
1621
1936
|
|
1622
1937
|
@property
|
1623
1938
|
def scheduler(self):
|
1939
|
+
"""
|
1940
|
+
Which scheduler to use.
|
1941
|
+
"""
|
1624
1942
|
return self._get_value("scheduler")
|
1625
1943
|
|
1626
1944
|
@scheduler.setter
|
@@ -1631,6 +1949,9 @@ class ResourceSpec(JSONLike):
|
|
1631
1949
|
|
1632
1950
|
@property
|
1633
1951
|
def shell(self):
|
1952
|
+
"""
|
1953
|
+
Which system shell to use.
|
1954
|
+
"""
|
1634
1955
|
return self._get_value("shell")
|
1635
1956
|
|
1636
1957
|
@shell.setter
|
@@ -1641,54 +1962,93 @@ class ResourceSpec(JSONLike):
|
|
1641
1962
|
|
1642
1963
|
@property
|
1643
1964
|
def use_job_array(self):
|
1965
|
+
"""
|
1966
|
+
Whether to use array jobs.
|
1967
|
+
"""
|
1644
1968
|
return self._get_value("use_job_array")
|
1645
1969
|
|
1646
1970
|
@property
|
1647
1971
|
def max_array_items(self):
|
1972
|
+
"""
|
1973
|
+
If using array jobs, up to how many items should be in the job array.
|
1974
|
+
"""
|
1648
1975
|
return self._get_value("max_array_items")
|
1649
1976
|
|
1650
1977
|
@property
|
1651
1978
|
def time_limit(self):
|
1979
|
+
"""
|
1980
|
+
How long to run for.
|
1981
|
+
"""
|
1652
1982
|
return self._get_value("time_limit")
|
1653
1983
|
|
1654
1984
|
@property
|
1655
1985
|
def scheduler_args(self):
|
1986
|
+
"""
|
1987
|
+
Additional arguments to pass to the scheduler.
|
1988
|
+
"""
|
1656
1989
|
return self._get_value("scheduler_args")
|
1657
1990
|
|
1658
1991
|
@property
|
1659
1992
|
def shell_args(self):
|
1993
|
+
"""
|
1994
|
+
Additional arguments to pass to the shell.
|
1995
|
+
"""
|
1660
1996
|
return self._get_value("shell_args")
|
1661
1997
|
|
1662
1998
|
@property
|
1663
1999
|
def os_name(self):
|
2000
|
+
"""
|
2001
|
+
Which OS to use.
|
2002
|
+
"""
|
1664
2003
|
return self._get_value("os_name")
|
1665
2004
|
|
1666
2005
|
@property
|
1667
2006
|
def environments(self):
|
2007
|
+
"""
|
2008
|
+
Which execution environments to use.
|
2009
|
+
"""
|
1668
2010
|
return self._get_value("environments")
|
1669
2011
|
|
1670
2012
|
@property
|
1671
2013
|
def SGE_parallel_env(self):
|
2014
|
+
"""
|
2015
|
+
Which SGE parallel environment to request.
|
2016
|
+
"""
|
1672
2017
|
return self._get_value("SGE_parallel_env")
|
1673
2018
|
|
1674
2019
|
@property
|
1675
2020
|
def SLURM_partition(self):
|
2021
|
+
"""
|
2022
|
+
Which SLURM partition to request.
|
2023
|
+
"""
|
1676
2024
|
return self._get_value("SLURM_partition")
|
1677
2025
|
|
1678
2026
|
@property
|
1679
2027
|
def SLURM_num_tasks(self):
|
2028
|
+
"""
|
2029
|
+
How many SLURM tasks to request.
|
2030
|
+
"""
|
1680
2031
|
return self._get_value("SLURM_num_tasks")
|
1681
2032
|
|
1682
2033
|
@property
|
1683
2034
|
def SLURM_num_tasks_per_node(self):
|
2035
|
+
"""
|
2036
|
+
How many SLURM tasks per compute node to request.
|
2037
|
+
"""
|
1684
2038
|
return self._get_value("SLURM_num_tasks_per_node")
|
1685
2039
|
|
1686
2040
|
@property
|
1687
2041
|
def SLURM_num_nodes(self):
|
2042
|
+
"""
|
2043
|
+
How many compute nodes to request.
|
2044
|
+
"""
|
1688
2045
|
return self._get_value("SLURM_num_nodes")
|
1689
2046
|
|
1690
2047
|
@property
|
1691
2048
|
def SLURM_num_cpus_per_task(self):
|
2049
|
+
"""
|
2050
|
+
How many CPU cores to ask for per SLURM task.
|
2051
|
+
"""
|
1692
2052
|
return self._get_value("SLURM_num_cpus_per_task")
|
1693
2053
|
|
1694
2054
|
@os_name.setter
|
@@ -1698,6 +2058,9 @@ class ResourceSpec(JSONLike):
|
|
1698
2058
|
|
1699
2059
|
@property
|
1700
2060
|
def workflow(self):
|
2061
|
+
"""
|
2062
|
+
The workflow owning this resource spec.
|
2063
|
+
"""
|
1701
2064
|
if self._workflow:
|
1702
2065
|
return self._workflow
|
1703
2066
|
|
@@ -1718,27 +2081,69 @@ class ResourceSpec(JSONLike):
|
|
1718
2081
|
|
1719
2082
|
@property
|
1720
2083
|
def element_set(self):
|
2084
|
+
"""
|
2085
|
+
The element set that will use this resource spec.
|
2086
|
+
"""
|
1721
2087
|
return self._resource_list.element_set
|
1722
2088
|
|
1723
2089
|
@property
|
1724
2090
|
def workflow_template(self):
|
2091
|
+
"""
|
2092
|
+
The workflow template that will use this resource spec.
|
2093
|
+
"""
|
1725
2094
|
return self._resource_list.workflow_template
|
1726
2095
|
|
1727
2096
|
|
1728
2097
|
class InputSourceType(enum.Enum):
|
2098
|
+
"""
|
2099
|
+
The types if input sources.
|
2100
|
+
"""
|
2101
|
+
|
2102
|
+
#: Input source is an import.
|
1729
2103
|
IMPORT = 0
|
2104
|
+
#: Input source is local.
|
1730
2105
|
LOCAL = 1
|
2106
|
+
#: Input source is a default.
|
1731
2107
|
DEFAULT = 2
|
2108
|
+
#: Input source is a task.
|
1732
2109
|
TASK = 3
|
1733
2110
|
|
1734
2111
|
|
1735
2112
|
class TaskSourceType(enum.Enum):
|
2113
|
+
"""
|
2114
|
+
The types of task-based input sources.
|
2115
|
+
"""
|
2116
|
+
|
2117
|
+
#: Input source is a task input.
|
1736
2118
|
INPUT = 0
|
2119
|
+
#: Input source is a task output.
|
1737
2120
|
OUTPUT = 1
|
2121
|
+
#: Input source is unspecified.
|
1738
2122
|
ANY = 2
|
1739
2123
|
|
1740
2124
|
|
1741
2125
|
class InputSource(JSONLike):
|
2126
|
+
"""
|
2127
|
+
An input source to a workflow task.
|
2128
|
+
|
2129
|
+
Parameters
|
2130
|
+
----------
|
2131
|
+
source_type: InputSourceType
|
2132
|
+
Type of the input source.
|
2133
|
+
import_ref:
|
2134
|
+
Where the input comes from when the type is `IMPORT`.
|
2135
|
+
task_ref:
|
2136
|
+
Which task is this an input for? Used when the type is `TASK`.
|
2137
|
+
task_source_type: TaskSourceType
|
2138
|
+
Type of task source.
|
2139
|
+
element_iters:
|
2140
|
+
Which element iterations does this apply to?
|
2141
|
+
path:
|
2142
|
+
Path to where this input goes.
|
2143
|
+
where: ~hpcflow.app.Rule | list[~hpcflow.app.Rule] | ~hpcflow.app.ElementFilter
|
2144
|
+
Filtering rules.
|
2145
|
+
"""
|
2146
|
+
|
1742
2147
|
_child_objects = (
|
1743
2148
|
ChildObjectSpec(
|
1744
2149
|
name="source_type",
|
@@ -1769,12 +2174,19 @@ class InputSource(JSONLike):
|
|
1769
2174
|
rules[idx] = app.Rule(**i)
|
1770
2175
|
where = app.ElementFilter(rules=rules)
|
1771
2176
|
|
2177
|
+
#: Type of the input source.
|
1772
2178
|
self.source_type = self._validate_source_type(source_type)
|
2179
|
+
#: Where the input comes from when the type is `IMPORT`.
|
1773
2180
|
self.import_ref = import_ref
|
2181
|
+
#: Which task is this an input for? Used when the type is `TASK`.
|
1774
2182
|
self.task_ref = task_ref
|
2183
|
+
#: Type of task source.
|
1775
2184
|
self.task_source_type = self._validate_task_source_type(task_source_type)
|
2185
|
+
#: Which element iterations does this apply to?
|
1776
2186
|
self.element_iters = element_iters
|
2187
|
+
#: Filtering rules.
|
1777
2188
|
self.where = where
|
2189
|
+
#: Path to where this input goes.
|
1778
2190
|
self.path = path
|
1779
2191
|
|
1780
2192
|
if self.source_type is InputSourceType.TASK:
|
@@ -1852,6 +2264,9 @@ class InputSource(JSONLike):
|
|
1852
2264
|
return None
|
1853
2265
|
|
1854
2266
|
def to_string(self):
|
2267
|
+
"""
|
2268
|
+
Render this input source as a string.
|
2269
|
+
"""
|
1855
2270
|
out = [self.source_type.name.lower()]
|
1856
2271
|
if self.source_type is InputSourceType.TASK:
|
1857
2272
|
out += [str(self.task_ref), self.task_source_type.name.lower()]
|
@@ -1893,20 +2308,26 @@ class InputSource(JSONLike):
|
|
1893
2308
|
|
1894
2309
|
@classmethod
|
1895
2310
|
def from_string(cls, str_defn):
|
1896
|
-
return cls(**cls._parse_from_string(str_defn))
|
1897
|
-
|
1898
|
-
@classmethod
|
1899
|
-
def _parse_from_string(cls, str_defn):
|
1900
2311
|
"""Parse a dot-delimited string definition of an InputSource.
|
1901
2312
|
|
1902
|
-
|
1903
|
-
|
1904
|
-
|
1905
|
-
|
1906
|
-
|
1907
|
-
|
2313
|
+
Parameter
|
2314
|
+
---------
|
2315
|
+
str_defn:
|
2316
|
+
The string to parse.
|
2317
|
+
|
2318
|
+
Examples
|
2319
|
+
--------
|
2320
|
+
task.[task_ref].input
|
2321
|
+
task.[task_ref].output
|
2322
|
+
local
|
2323
|
+
default
|
2324
|
+
import.[import_ref]
|
1908
2325
|
|
1909
2326
|
"""
|
2327
|
+
return cls(**cls._parse_from_string(str_defn))
|
2328
|
+
|
2329
|
+
@classmethod
|
2330
|
+
def _parse_from_string(cls, str_defn):
|
1910
2331
|
parts = str_defn.split(".")
|
1911
2332
|
source_type = cls._validate_source_type(parts[0])
|
1912
2333
|
task_ref = None
|
@@ -1957,6 +2378,18 @@ class InputSource(JSONLike):
|
|
1957
2378
|
|
1958
2379
|
@classmethod
|
1959
2380
|
def import_(cls, import_ref, element_iters=None, where=None):
|
2381
|
+
"""
|
2382
|
+
Make an instnace of an input source that is an import.
|
2383
|
+
|
2384
|
+
Parameters
|
2385
|
+
----------
|
2386
|
+
import_ref:
|
2387
|
+
Import reference.
|
2388
|
+
element_iters:
|
2389
|
+
Originating element iterations.
|
2390
|
+
where:
|
2391
|
+
Filtering rule.
|
2392
|
+
"""
|
1960
2393
|
return cls(
|
1961
2394
|
source_type=cls.app.InputSourceType.IMPORT,
|
1962
2395
|
import_ref=import_ref,
|
@@ -1966,14 +2399,34 @@ class InputSource(JSONLike):
|
|
1966
2399
|
|
1967
2400
|
@classmethod
|
1968
2401
|
def local(cls):
|
2402
|
+
"""
|
2403
|
+
Make an instnace of an input source that is local.
|
2404
|
+
"""
|
1969
2405
|
return cls(source_type=cls.app.InputSourceType.LOCAL)
|
1970
2406
|
|
1971
2407
|
@classmethod
|
1972
2408
|
def default(cls):
|
2409
|
+
"""
|
2410
|
+
Make an instnace of an input source that is default.
|
2411
|
+
"""
|
1973
2412
|
return cls(source_type=cls.app.InputSourceType.DEFAULT)
|
1974
2413
|
|
1975
2414
|
@classmethod
|
1976
2415
|
def task(cls, task_ref, task_source_type=None, element_iters=None, where=None):
|
2416
|
+
"""
|
2417
|
+
Make an instnace of an input source that is a task.
|
2418
|
+
|
2419
|
+
Parameters
|
2420
|
+
----------
|
2421
|
+
task_ref:
|
2422
|
+
Source task reference.
|
2423
|
+
task_source_type:
|
2424
|
+
Type of task source.
|
2425
|
+
element_iters:
|
2426
|
+
Originating element iterations.
|
2427
|
+
where:
|
2428
|
+
Filtering rule.
|
2429
|
+
"""
|
1977
2430
|
if not task_source_type:
|
1978
2431
|
task_source_type = cls.app.TaskSourceType.OUTPUT
|
1979
2432
|
return cls(
|