hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a181__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/_version.py +1 -1
- hpcflow/data/demo_data_manifest/__init__.py +3 -0
- hpcflow/sdk/__init__.py +4 -1
- hpcflow/sdk/app.py +160 -15
- hpcflow/sdk/cli.py +14 -0
- hpcflow/sdk/cli_common.py +83 -0
- hpcflow/sdk/config/__init__.py +4 -0
- hpcflow/sdk/config/callbacks.py +25 -2
- hpcflow/sdk/config/cli.py +4 -1
- hpcflow/sdk/config/config.py +188 -14
- hpcflow/sdk/config/config_file.py +91 -3
- hpcflow/sdk/config/errors.py +33 -0
- hpcflow/sdk/core/__init__.py +2 -0
- hpcflow/sdk/core/actions.py +492 -35
- hpcflow/sdk/core/cache.py +22 -0
- hpcflow/sdk/core/command_files.py +221 -5
- hpcflow/sdk/core/commands.py +57 -0
- hpcflow/sdk/core/element.py +407 -8
- hpcflow/sdk/core/environment.py +92 -0
- hpcflow/sdk/core/errors.py +245 -61
- hpcflow/sdk/core/json_like.py +72 -14
- hpcflow/sdk/core/loop.py +122 -21
- hpcflow/sdk/core/loop_cache.py +34 -9
- hpcflow/sdk/core/object_list.py +172 -26
- hpcflow/sdk/core/parallel.py +14 -0
- hpcflow/sdk/core/parameters.py +478 -25
- hpcflow/sdk/core/rule.py +31 -1
- hpcflow/sdk/core/run_dir_files.py +12 -2
- hpcflow/sdk/core/task.py +407 -80
- hpcflow/sdk/core/task_schema.py +70 -9
- hpcflow/sdk/core/test_utils.py +35 -0
- hpcflow/sdk/core/utils.py +101 -4
- hpcflow/sdk/core/validation.py +13 -1
- hpcflow/sdk/core/workflow.py +316 -96
- hpcflow/sdk/core/zarr_io.py +23 -0
- hpcflow/sdk/data/__init__.py +13 -0
- hpcflow/sdk/demo/__init__.py +3 -0
- hpcflow/sdk/helper/__init__.py +3 -0
- hpcflow/sdk/helper/cli.py +9 -0
- hpcflow/sdk/helper/helper.py +28 -0
- hpcflow/sdk/helper/watcher.py +33 -0
- hpcflow/sdk/log.py +40 -0
- hpcflow/sdk/persistence/__init__.py +14 -4
- hpcflow/sdk/persistence/base.py +289 -23
- hpcflow/sdk/persistence/json.py +29 -0
- hpcflow/sdk/persistence/pending.py +217 -107
- hpcflow/sdk/persistence/store_resource.py +58 -2
- hpcflow/sdk/persistence/utils.py +8 -0
- hpcflow/sdk/persistence/zarr.py +68 -1
- hpcflow/sdk/runtime.py +52 -10
- hpcflow/sdk/submission/__init__.py +3 -0
- hpcflow/sdk/submission/jobscript.py +198 -9
- hpcflow/sdk/submission/jobscript_info.py +13 -0
- hpcflow/sdk/submission/schedulers/__init__.py +60 -0
- hpcflow/sdk/submission/schedulers/direct.py +53 -0
- hpcflow/sdk/submission/schedulers/sge.py +45 -7
- hpcflow/sdk/submission/schedulers/slurm.py +45 -8
- hpcflow/sdk/submission/schedulers/utils.py +4 -0
- hpcflow/sdk/submission/shells/__init__.py +11 -1
- hpcflow/sdk/submission/shells/base.py +32 -1
- hpcflow/sdk/submission/shells/bash.py +36 -1
- hpcflow/sdk/submission/shells/os_version.py +18 -6
- hpcflow/sdk/submission/shells/powershell.py +22 -0
- hpcflow/sdk/submission/submission.py +88 -3
- hpcflow/sdk/typing.py +10 -1
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/METADATA +3 -3
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/RECORD +70 -70
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/core/actions.py
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
"""
|
2
|
+
Actions are base components of elements.
|
3
|
+
Element action runs (EARs) are the basic components of any enactment;
|
4
|
+
they may be grouped together within a jobscript for efficiency.
|
5
|
+
"""
|
6
|
+
|
1
7
|
from __future__ import annotations
|
2
8
|
import copy
|
3
9
|
from dataclasses import dataclass
|
@@ -37,13 +43,23 @@ ACTION_SCOPE_REGEX = r"(\w*)(?:\[(.*)\])?"
|
|
37
43
|
|
38
44
|
|
39
45
|
class ActionScopeType(enum.Enum):
|
46
|
+
"""
|
47
|
+
Types of action scope.
|
48
|
+
"""
|
49
|
+
|
50
|
+
#: Scope that applies to anything.
|
40
51
|
ANY = 0
|
52
|
+
#: Scope that only applies to main scripts.
|
41
53
|
MAIN = 1
|
54
|
+
#: Scope that applies to processing steps.
|
42
55
|
PROCESSING = 2
|
56
|
+
#: Scope that applies to input file generators.
|
43
57
|
INPUT_FILE_GENERATOR = 3
|
58
|
+
#: Scope that applies to output file parsers.
|
44
59
|
OUTPUT_FILE_PARSER = 4
|
45
60
|
|
46
61
|
|
62
|
+
#: Keyword arguments permitted for particular scopes.
|
47
63
|
ACTION_SCOPE_ALLOWED_KWARGS = {
|
48
64
|
ActionScopeType.ANY.name: set(),
|
49
65
|
ActionScopeType.MAIN.name: set(),
|
@@ -64,30 +80,36 @@ class EARStatus(enum.Enum):
|
|
64
80
|
member.__doc__ = doc
|
65
81
|
return member
|
66
82
|
|
83
|
+
#: Not yet associated with a submission.
|
67
84
|
pending = (
|
68
85
|
0,
|
69
86
|
".",
|
70
87
|
"grey46",
|
71
88
|
"Not yet associated with a submission.",
|
72
89
|
)
|
90
|
+
#: Associated with a prepared submission that is not yet submitted.
|
73
91
|
prepared = (
|
74
92
|
1,
|
75
93
|
".",
|
76
94
|
"grey46",
|
77
95
|
"Associated with a prepared submission that is not yet submitted.",
|
78
96
|
)
|
97
|
+
#: Submitted for execution.
|
79
98
|
submitted = (
|
80
99
|
2,
|
81
100
|
".",
|
82
101
|
"grey46",
|
83
102
|
"Submitted for execution.",
|
84
103
|
)
|
104
|
+
#: Executing now.
|
85
105
|
running = (
|
86
106
|
3,
|
87
107
|
"●",
|
88
108
|
"dodger_blue1",
|
89
109
|
"Executing now.",
|
90
110
|
)
|
111
|
+
#: Not attempted due to a failure of an upstream action on which this depends,
|
112
|
+
#: or a loop termination condition being satisfied.
|
91
113
|
skipped = (
|
92
114
|
4,
|
93
115
|
"s",
|
@@ -97,18 +119,21 @@ class EARStatus(enum.Enum):
|
|
97
119
|
"or a loop termination condition being satisfied."
|
98
120
|
),
|
99
121
|
)
|
122
|
+
#: Aborted by the user; downstream actions will be attempted.
|
100
123
|
aborted = (
|
101
124
|
5,
|
102
125
|
"A",
|
103
126
|
"deep_pink4",
|
104
127
|
"Aborted by the user; downstream actions will be attempted.",
|
105
128
|
)
|
129
|
+
#: Probably exited successfully.
|
106
130
|
success = (
|
107
131
|
6,
|
108
132
|
"■",
|
109
133
|
"green3",
|
110
134
|
"Probably exited successfully.",
|
111
135
|
)
|
136
|
+
#: Probably failed.
|
112
137
|
error = (
|
113
138
|
7,
|
114
139
|
"E",
|
@@ -128,10 +153,57 @@ class EARStatus(enum.Enum):
|
|
128
153
|
|
129
154
|
@property
|
130
155
|
def rich_repr(self):
|
156
|
+
"""
|
157
|
+
The rich representation of the value.
|
158
|
+
"""
|
131
159
|
return f"[{self.colour}]{self.symbol}[/{self.colour}]"
|
132
160
|
|
133
161
|
|
134
162
|
class ElementActionRun:
|
163
|
+
"""
|
164
|
+
The Element Action Run (EAR) is an atomic unit of an enacted workflow, representing
|
165
|
+
one unit of work (e.g., particular submitted job to run a program) within that
|
166
|
+
overall workflow. With looping over, say, parameter spaces, there may be many EARs
|
167
|
+
per element.
|
168
|
+
|
169
|
+
Parameters
|
170
|
+
----------
|
171
|
+
id_: int
|
172
|
+
The ID of the EAR.
|
173
|
+
is_pending: bool
|
174
|
+
Whether this EAR is pending.
|
175
|
+
element_action:
|
176
|
+
The particular element action that this is a run of.
|
177
|
+
index: int:
|
178
|
+
The index of the run within the collection of runs.
|
179
|
+
data_idx: dict
|
180
|
+
Used for looking up input data to the EAR.
|
181
|
+
commands_idx: list[int]
|
182
|
+
Indices of commands to apply.
|
183
|
+
start_time: datetime
|
184
|
+
Time of start of run, if the run has ever been started.
|
185
|
+
end_time: datetime
|
186
|
+
Time of end of run, if the run has ever ended.
|
187
|
+
snapshot_start: dict
|
188
|
+
Parameters for taking a snapshot of the data directory before the run.
|
189
|
+
If unspecified, no snapshot will be taken.
|
190
|
+
snapshot_end: dict
|
191
|
+
Parameters for taking a snapshot of the data directory after the run.
|
192
|
+
If unspecified, no snapshot will be taken.
|
193
|
+
submission_idx: int
|
194
|
+
What submission was this (if it has been submitted)?
|
195
|
+
success: bool
|
196
|
+
Whether this EAR succeeded (if it has run).
|
197
|
+
skip: bool
|
198
|
+
Whether this EAR was skipped.
|
199
|
+
exit_code: int
|
200
|
+
The exit code, if known.
|
201
|
+
metadata: dict
|
202
|
+
Metadata about the EAR.
|
203
|
+
run_hostname: str
|
204
|
+
Where to run the EAR (if not locally).
|
205
|
+
"""
|
206
|
+
|
135
207
|
_app_attr = "app"
|
136
208
|
|
137
209
|
def __init__(
|
@@ -189,14 +261,23 @@ class ElementActionRun:
|
|
189
261
|
|
190
262
|
@property
|
191
263
|
def id_(self) -> int:
|
264
|
+
"""
|
265
|
+
The ID of the EAR.
|
266
|
+
"""
|
192
267
|
return self._id
|
193
268
|
|
194
269
|
@property
|
195
270
|
def is_pending(self) -> bool:
|
271
|
+
"""
|
272
|
+
Whether this EAR is pending.
|
273
|
+
"""
|
196
274
|
return self._is_pending
|
197
275
|
|
198
276
|
@property
|
199
277
|
def element_action(self):
|
278
|
+
"""
|
279
|
+
The particular element action that this is a run of.
|
280
|
+
"""
|
200
281
|
return self._element_action
|
201
282
|
|
202
283
|
@property
|
@@ -206,58 +287,100 @@ class ElementActionRun:
|
|
206
287
|
|
207
288
|
@property
|
208
289
|
def action(self):
|
290
|
+
"""
|
291
|
+
The action this is a run of.
|
292
|
+
"""
|
209
293
|
return self.element_action.action
|
210
294
|
|
211
295
|
@property
|
212
296
|
def element_iteration(self):
|
297
|
+
"""
|
298
|
+
The iteration information of this run.
|
299
|
+
"""
|
213
300
|
return self.element_action.element_iteration
|
214
301
|
|
215
302
|
@property
|
216
303
|
def element(self):
|
304
|
+
"""
|
305
|
+
The element this is a run of.
|
306
|
+
"""
|
217
307
|
return self.element_iteration.element
|
218
308
|
|
219
309
|
@property
|
220
310
|
def workflow(self):
|
311
|
+
"""
|
312
|
+
The workflow this is a run of.
|
313
|
+
"""
|
221
314
|
return self.element_iteration.workflow
|
222
315
|
|
223
316
|
@property
|
224
317
|
def data_idx(self):
|
318
|
+
"""
|
319
|
+
Used for looking up input data to the EAR.
|
320
|
+
"""
|
225
321
|
return self._data_idx
|
226
322
|
|
227
323
|
@property
|
228
324
|
def commands_idx(self):
|
325
|
+
"""
|
326
|
+
Indices of commands to apply.
|
327
|
+
"""
|
229
328
|
return self._commands_idx
|
230
329
|
|
231
330
|
@property
|
232
331
|
def metadata(self):
|
332
|
+
"""
|
333
|
+
Metadata about the EAR.
|
334
|
+
"""
|
233
335
|
return self._metadata
|
234
336
|
|
235
337
|
@property
|
236
338
|
def run_hostname(self):
|
339
|
+
"""
|
340
|
+
Where to run the EAR, if known/specified.
|
341
|
+
"""
|
237
342
|
return self._run_hostname
|
238
343
|
|
239
344
|
@property
|
240
345
|
def start_time(self):
|
346
|
+
"""
|
347
|
+
When the EAR started.
|
348
|
+
"""
|
241
349
|
return self._start_time
|
242
350
|
|
243
351
|
@property
|
244
352
|
def end_time(self):
|
353
|
+
"""
|
354
|
+
When the EAR finished.
|
355
|
+
"""
|
245
356
|
return self._end_time
|
246
357
|
|
247
358
|
@property
|
248
359
|
def submission_idx(self):
|
360
|
+
"""
|
361
|
+
What actual submission index was this?
|
362
|
+
"""
|
249
363
|
return self._submission_idx
|
250
364
|
|
251
365
|
@property
|
252
366
|
def success(self):
|
367
|
+
"""
|
368
|
+
Did the EAR succeed?
|
369
|
+
"""
|
253
370
|
return self._success
|
254
371
|
|
255
372
|
@property
|
256
373
|
def skip(self):
|
374
|
+
"""
|
375
|
+
Was the EAR skipped?
|
376
|
+
"""
|
257
377
|
return self._skip
|
258
378
|
|
259
379
|
@property
|
260
380
|
def snapshot_start(self):
|
381
|
+
"""
|
382
|
+
The snapshot of the data directory at the start of the run.
|
383
|
+
"""
|
261
384
|
if self._ss_start_obj is None and self._snapshot_start:
|
262
385
|
self._ss_start_obj = JSONLikeDirSnapShot(
|
263
386
|
root_path=".",
|
@@ -267,14 +390,18 @@ class ElementActionRun:
|
|
267
390
|
|
268
391
|
@property
|
269
392
|
def snapshot_end(self):
|
393
|
+
"""
|
394
|
+
The snapshot of the data directory at the end of the run.
|
395
|
+
"""
|
270
396
|
if self._ss_end_obj is None and self._snapshot_end:
|
271
397
|
self._ss_end_obj = JSONLikeDirSnapShot(root_path=".", **self._snapshot_end)
|
272
398
|
return self._ss_end_obj
|
273
399
|
|
274
400
|
@property
|
275
401
|
def dir_diff(self) -> DirectorySnapshotDiff:
|
276
|
-
"""
|
277
|
-
EAR.
|
402
|
+
"""
|
403
|
+
The changes to the EAR working directory due to the execution of this EAR.
|
404
|
+
"""
|
278
405
|
if self._ss_diff_obj is None and self.snapshot_end:
|
279
406
|
self._ss_diff_obj = DirectorySnapshotDiff(
|
280
407
|
self.snapshot_start, self.snapshot_end
|
@@ -283,15 +410,23 @@ class ElementActionRun:
|
|
283
410
|
|
284
411
|
@property
|
285
412
|
def exit_code(self):
|
413
|
+
"""
|
414
|
+
The exit code of the underlying program run by the EAR, if known.
|
415
|
+
"""
|
286
416
|
return self._exit_code
|
287
417
|
|
288
418
|
@property
|
289
419
|
def task(self):
|
420
|
+
"""
|
421
|
+
The task that this EAR is part of the implementation of.
|
422
|
+
"""
|
290
423
|
return self.element_action.task
|
291
424
|
|
292
425
|
@property
|
293
426
|
def status(self):
|
294
|
-
"""
|
427
|
+
"""
|
428
|
+
The state of this EAR.
|
429
|
+
"""
|
295
430
|
|
296
431
|
if self.skip:
|
297
432
|
return EARStatus.skipped
|
@@ -336,6 +471,14 @@ class ElementActionRun:
|
|
336
471
|
return self.action.get_parameter_names(prefix)
|
337
472
|
|
338
473
|
def get_data_idx(self, path: str = None):
|
474
|
+
"""
|
475
|
+
Get the data index of a value in the most recent iteration.
|
476
|
+
|
477
|
+
Parameters
|
478
|
+
----------
|
479
|
+
path:
|
480
|
+
Path to the parameter.
|
481
|
+
"""
|
339
482
|
return self.element_iteration.get_data_idx(
|
340
483
|
path,
|
341
484
|
action_idx=self.element_action.action_idx,
|
@@ -350,6 +493,20 @@ class ElementActionRun:
|
|
350
493
|
as_strings: bool = False,
|
351
494
|
use_task_index: bool = False,
|
352
495
|
):
|
496
|
+
"""
|
497
|
+
Get the source or sources of a parameter in the most recent iteration.
|
498
|
+
|
499
|
+
Parameters
|
500
|
+
----------
|
501
|
+
path:
|
502
|
+
Path to the parameter.
|
503
|
+
typ:
|
504
|
+
The parameter type.
|
505
|
+
as_strings:
|
506
|
+
Whether to return the result as human-readable strings.
|
507
|
+
use_task_index:
|
508
|
+
Whether to use the task index.
|
509
|
+
"""
|
353
510
|
return self.element_iteration.get_parameter_sources(
|
354
511
|
path,
|
355
512
|
action_idx=self.element_action.action_idx,
|
@@ -366,6 +523,22 @@ class ElementActionRun:
|
|
366
523
|
raise_on_missing: bool = False,
|
367
524
|
raise_on_unset: bool = False,
|
368
525
|
):
|
526
|
+
"""
|
527
|
+
Get a value (parameter, input, output, etc.) from the most recent iteration.
|
528
|
+
|
529
|
+
Parameters
|
530
|
+
----------
|
531
|
+
path:
|
532
|
+
Path to the value.
|
533
|
+
default:
|
534
|
+
Default value to provide if value absent.
|
535
|
+
raise_on_missing:
|
536
|
+
Whether to raise an exception on an absent value.
|
537
|
+
If not, the default is returned.
|
538
|
+
raise_on_unset:
|
539
|
+
Whether to raise an exception on an explicitly unset value.
|
540
|
+
If not, the default is returned.
|
541
|
+
"""
|
369
542
|
return self.element_iteration.get(
|
370
543
|
path=path,
|
371
544
|
action_idx=self.element_action.action_idx,
|
@@ -436,12 +609,18 @@ class ElementActionRun:
|
|
436
609
|
|
437
610
|
@property
|
438
611
|
def inputs(self):
|
612
|
+
"""
|
613
|
+
The inputs to this EAR.
|
614
|
+
"""
|
439
615
|
if not self._inputs:
|
440
616
|
self._inputs = self.app.ElementInputs(element_action_run=self)
|
441
617
|
return self._inputs
|
442
618
|
|
443
619
|
@property
|
444
620
|
def outputs(self):
|
621
|
+
"""
|
622
|
+
The outputs from this EAR.
|
623
|
+
"""
|
445
624
|
if not self._outputs:
|
446
625
|
self._outputs = self.app.ElementOutputs(element_action_run=self)
|
447
626
|
return self._outputs
|
@@ -449,24 +628,36 @@ class ElementActionRun:
|
|
449
628
|
@property
|
450
629
|
@TimeIt.decorator
|
451
630
|
def resources(self):
|
631
|
+
"""
|
632
|
+
The resources to use with (or used by) this EAR.
|
633
|
+
"""
|
452
634
|
if not self._resources:
|
453
635
|
self._resources = self.app.ElementResources(**self.get_resources())
|
454
636
|
return self._resources
|
455
637
|
|
456
638
|
@property
|
457
639
|
def input_files(self):
|
640
|
+
"""
|
641
|
+
The input files to the controlled program.
|
642
|
+
"""
|
458
643
|
if not self._input_files:
|
459
644
|
self._input_files = self.app.ElementInputFiles(element_action_run=self)
|
460
645
|
return self._input_files
|
461
646
|
|
462
647
|
@property
|
463
648
|
def output_files(self):
|
649
|
+
"""
|
650
|
+
The output files from the controlled program.
|
651
|
+
"""
|
464
652
|
if not self._output_files:
|
465
653
|
self._output_files = self.app.ElementOutputFiles(element_action_run=self)
|
466
654
|
return self._output_files
|
467
655
|
|
468
656
|
@property
|
469
657
|
def env_spec(self) -> Dict[str, Any]:
|
658
|
+
"""
|
659
|
+
Environment details.
|
660
|
+
"""
|
470
661
|
return self.resources.environments[self.action.get_environment_name()]
|
471
662
|
|
472
663
|
@TimeIt.decorator
|
@@ -476,9 +667,15 @@ class ElementActionRun:
|
|
476
667
|
return self.element_iteration.get_resources(self.action)
|
477
668
|
|
478
669
|
def get_environment_spec(self) -> str:
|
670
|
+
"""
|
671
|
+
What environment to run in?
|
672
|
+
"""
|
479
673
|
return self.action.get_environment_spec()
|
480
674
|
|
481
675
|
def get_environment(self) -> app.Environment:
|
676
|
+
"""
|
677
|
+
What environment to run in?
|
678
|
+
"""
|
482
679
|
return self.action.get_environment()
|
483
680
|
|
484
681
|
def get_all_previous_iteration_runs(self, include_self: bool = True):
|
@@ -502,13 +699,13 @@ class ElementActionRun:
|
|
502
699
|
|
503
700
|
Parameters
|
504
701
|
----------
|
505
|
-
inputs
|
702
|
+
inputs:
|
506
703
|
If specified, a list of input parameter types to include, or a dict whose keys
|
507
704
|
are input parameter types to include. For schema inputs that have
|
508
705
|
`multiple=True`, the input type should be labelled. If a dict is passed, and
|
509
706
|
the key "all_iterations` is present and `True`, the return for that input
|
510
707
|
will be structured to include values for all previous iterations.
|
511
|
-
label_dict
|
708
|
+
label_dict:
|
512
709
|
If True, arrange the values of schema inputs with multiple=True as a dict
|
513
710
|
whose keys are the labels. If False, labels will be included in the top level
|
514
711
|
keys.
|
@@ -565,6 +762,9 @@ class ElementActionRun:
|
|
565
762
|
return self.get_input_values(inputs=inputs, label_dict=label_dict)
|
566
763
|
|
567
764
|
def get_IFG_input_values(self) -> Dict[str, Any]:
|
765
|
+
"""
|
766
|
+
Get a dict of input values that are to be passed via an input file generator.
|
767
|
+
"""
|
568
768
|
if not self.action._from_expand:
|
569
769
|
raise RuntimeError(
|
570
770
|
f"Cannot get input file generator inputs from this EAR because the "
|
@@ -583,6 +783,10 @@ class ElementActionRun:
|
|
583
783
|
return inputs
|
584
784
|
|
585
785
|
def get_OFP_output_files(self) -> Dict[str, Union[str, List[str]]]:
|
786
|
+
"""
|
787
|
+
Get a dict of output files that are going to be parsed to generate one or more
|
788
|
+
outputs.
|
789
|
+
"""
|
586
790
|
# TODO: can this return multiple files for a given FileSpec?
|
587
791
|
if not self.action._from_expand:
|
588
792
|
raise RuntimeError(
|
@@ -595,6 +799,9 @@ class ElementActionRun:
|
|
595
799
|
return out_files
|
596
800
|
|
597
801
|
def get_OFP_inputs(self) -> Dict[str, Union[str, List[str]]]:
|
802
|
+
"""
|
803
|
+
Get a dict of input values that are to be passed to output file parsers.
|
804
|
+
"""
|
598
805
|
if not self.action._from_expand:
|
599
806
|
raise RuntimeError(
|
600
807
|
f"Cannot get output file parser inputs from this from EAR because the "
|
@@ -610,6 +817,9 @@ class ElementActionRun:
|
|
610
817
|
return inputs
|
611
818
|
|
612
819
|
def get_OFP_outputs(self) -> Dict[str, Union[str, List[str]]]:
|
820
|
+
"""
|
821
|
+
Get the outputs obtained by parsing an output file.
|
822
|
+
"""
|
613
823
|
if not self.action._from_expand:
|
614
824
|
raise RuntimeError(
|
615
825
|
f"Cannot get output file parser outputs from this from EAR because the "
|
@@ -621,6 +831,9 @@ class ElementActionRun:
|
|
621
831
|
return outputs
|
622
832
|
|
623
833
|
def write_source(self, js_idx: int, js_act_idx: int):
|
834
|
+
"""
|
835
|
+
Write values to files in standard formats.
|
836
|
+
"""
|
624
837
|
import h5py
|
625
838
|
|
626
839
|
for fmt, ins in self.action.script_data_in_grouped.items():
|
@@ -690,10 +903,13 @@ class ElementActionRun:
|
|
690
903
|
self, jobscript: app.Jobscript, JS_action_idx: int
|
691
904
|
) -> Tuple[str, List[str], List[int]]:
|
692
905
|
"""
|
906
|
+
Write the EAR's enactment to disk in preparation for submission.
|
907
|
+
|
693
908
|
Returns
|
694
909
|
-------
|
695
|
-
commands
|
696
|
-
|
910
|
+
commands:
|
911
|
+
List of argument words for the command that enacts the EAR.
|
912
|
+
shell_vars:
|
697
913
|
Dict whose keys are command indices, and whose values are lists of tuples,
|
698
914
|
where each tuple contains: (parameter name, shell variable name,
|
699
915
|
"stdout"/"stderr").
|
@@ -735,6 +951,20 @@ class ElementActionRun:
|
|
735
951
|
|
736
952
|
|
737
953
|
class ElementAction:
|
954
|
+
"""
|
955
|
+
An abstract representation of an element's action at a particular iteration and
|
956
|
+
the runs that enact that element iteration.
|
957
|
+
|
958
|
+
Parameters
|
959
|
+
----------
|
960
|
+
element_iteration:
|
961
|
+
The iteration
|
962
|
+
action_idx:
|
963
|
+
The action index.
|
964
|
+
runs:
|
965
|
+
The list of run indices.
|
966
|
+
"""
|
967
|
+
|
738
968
|
_app_attr = "app"
|
739
969
|
|
740
970
|
def __init__(self, element_iteration, action_idx, runs):
|
@@ -761,18 +991,30 @@ class ElementAction:
|
|
761
991
|
|
762
992
|
@property
|
763
993
|
def element_iteration(self):
|
994
|
+
"""
|
995
|
+
The iteration for this action.
|
996
|
+
"""
|
764
997
|
return self._element_iteration
|
765
998
|
|
766
999
|
@property
|
767
1000
|
def element(self):
|
1001
|
+
"""
|
1002
|
+
The element for this action.
|
1003
|
+
"""
|
768
1004
|
return self.element_iteration.element
|
769
1005
|
|
770
1006
|
@property
|
771
1007
|
def num_runs(self):
|
1008
|
+
"""
|
1009
|
+
The number of runs associated with this action.
|
1010
|
+
"""
|
772
1011
|
return len(self._runs)
|
773
1012
|
|
774
1013
|
@property
|
775
1014
|
def runs(self):
|
1015
|
+
"""
|
1016
|
+
The EARs that this action is enacted by.
|
1017
|
+
"""
|
776
1018
|
if self._run_objs is None:
|
777
1019
|
self._run_objs = [
|
778
1020
|
self.app.ElementActionRun(
|
@@ -790,41 +1032,65 @@ class ElementAction:
|
|
790
1032
|
|
791
1033
|
@property
|
792
1034
|
def task(self):
|
1035
|
+
"""
|
1036
|
+
The task that this action is an instance of.
|
1037
|
+
"""
|
793
1038
|
return self.element_iteration.task
|
794
1039
|
|
795
1040
|
@property
|
796
1041
|
def action_idx(self):
|
1042
|
+
"""
|
1043
|
+
The index of the action.
|
1044
|
+
"""
|
797
1045
|
return self._action_idx
|
798
1046
|
|
799
1047
|
@property
|
800
1048
|
def action(self):
|
1049
|
+
"""
|
1050
|
+
The abstract task that this is a concrete model of.
|
1051
|
+
"""
|
801
1052
|
return self.task.template.get_schema_action(self.action_idx)
|
802
1053
|
|
803
1054
|
@property
|
804
1055
|
def inputs(self):
|
1056
|
+
"""
|
1057
|
+
The inputs to this action.
|
1058
|
+
"""
|
805
1059
|
if not self._inputs:
|
806
1060
|
self._inputs = self.app.ElementInputs(element_action=self)
|
807
1061
|
return self._inputs
|
808
1062
|
|
809
1063
|
@property
|
810
1064
|
def outputs(self):
|
1065
|
+
"""
|
1066
|
+
The outputs from this action.
|
1067
|
+
"""
|
811
1068
|
if not self._outputs:
|
812
1069
|
self._outputs = self.app.ElementOutputs(element_action=self)
|
813
1070
|
return self._outputs
|
814
1071
|
|
815
1072
|
@property
|
816
1073
|
def input_files(self):
|
1074
|
+
"""
|
1075
|
+
The input files to this action.
|
1076
|
+
"""
|
817
1077
|
if not self._input_files:
|
818
1078
|
self._input_files = self.app.ElementInputFiles(element_action=self)
|
819
1079
|
return self._input_files
|
820
1080
|
|
821
1081
|
@property
|
822
1082
|
def output_files(self):
|
1083
|
+
"""
|
1084
|
+
The output files from this action.
|
1085
|
+
"""
|
823
1086
|
if not self._output_files:
|
824
1087
|
self._output_files = self.app.ElementOutputFiles(element_action=self)
|
825
1088
|
return self._output_files
|
826
1089
|
|
827
1090
|
def get_data_idx(self, path: str = None, run_idx: int = -1):
|
1091
|
+
"""
|
1092
|
+
Get the data index for some path/run.
|
1093
|
+
"""
|
828
1094
|
return self.element_iteration.get_data_idx(
|
829
1095
|
path,
|
830
1096
|
action_idx=self.action_idx,
|
@@ -839,6 +1105,9 @@ class ElementAction:
|
|
839
1105
|
as_strings: bool = False,
|
840
1106
|
use_task_index: bool = False,
|
841
1107
|
):
|
1108
|
+
"""
|
1109
|
+
Get information about where parameters originated.
|
1110
|
+
"""
|
842
1111
|
return self.element_iteration.get_parameter_sources(
|
843
1112
|
path,
|
844
1113
|
action_idx=self.action_idx,
|
@@ -856,6 +1125,9 @@ class ElementAction:
|
|
856
1125
|
raise_on_missing: bool = False,
|
857
1126
|
raise_on_unset: bool = False,
|
858
1127
|
):
|
1128
|
+
"""
|
1129
|
+
Get the value of a parameter.
|
1130
|
+
"""
|
859
1131
|
return self.element_iteration.get(
|
860
1132
|
path=path,
|
861
1133
|
action_idx=self.action_idx,
|
@@ -868,8 +1140,8 @@ class ElementAction:
|
|
868
1140
|
def get_parameter_names(self, prefix: str) -> List[str]:
|
869
1141
|
"""Get parameter types associated with a given prefix.
|
870
1142
|
|
871
|
-
For inputs, labels are ignored.
|
872
|
-
information.
|
1143
|
+
For inputs, labels are ignored.
|
1144
|
+
See :py:meth:`.Action.get_parameter_names` for more information.
|
873
1145
|
|
874
1146
|
Parameters
|
875
1147
|
----------
|
@@ -898,7 +1170,9 @@ class ActionScope(JSONLike):
|
|
898
1170
|
if isinstance(typ, str):
|
899
1171
|
typ = getattr(self.app.ActionScopeType, typ.upper())
|
900
1172
|
|
1173
|
+
#: Action scope type.
|
901
1174
|
self.typ = typ
|
1175
|
+
#: Any provided extra keyword arguments.
|
902
1176
|
self.kwargs = {k: v for k, v in kwargs.items() if v is not None}
|
903
1177
|
|
904
1178
|
bad_keys = set(kwargs.keys()) - ACTION_SCOPE_ALLOWED_KWARGS[self.typ.name]
|
@@ -932,6 +1206,9 @@ class ActionScope(JSONLike):
|
|
932
1206
|
return {"type": typ_str, **kwargs}
|
933
1207
|
|
934
1208
|
def to_string(self):
|
1209
|
+
"""
|
1210
|
+
Render this action scope as a string.
|
1211
|
+
"""
|
935
1212
|
kwargs_str = ""
|
936
1213
|
if self.kwargs:
|
937
1214
|
kwargs_str = "[" + ", ".join(f"{k}={v}" for k, v in self.kwargs.items()) + "]"
|
@@ -948,27 +1225,46 @@ class ActionScope(JSONLike):
|
|
948
1225
|
|
949
1226
|
@classmethod
|
950
1227
|
def any(cls):
|
1228
|
+
"""
|
1229
|
+
Any scope.
|
1230
|
+
"""
|
951
1231
|
return cls(typ=ActionScopeType.ANY)
|
952
1232
|
|
953
1233
|
@classmethod
|
954
1234
|
def main(cls):
|
1235
|
+
"""
|
1236
|
+
The main scope.
|
1237
|
+
"""
|
955
1238
|
return cls(typ=ActionScopeType.MAIN)
|
956
1239
|
|
957
1240
|
@classmethod
|
958
1241
|
def processing(cls):
|
1242
|
+
"""
|
1243
|
+
The processing scope.
|
1244
|
+
"""
|
959
1245
|
return cls(typ=ActionScopeType.PROCESSING)
|
960
1246
|
|
961
1247
|
@classmethod
|
962
1248
|
def input_file_generator(cls, file=None):
|
1249
|
+
"""
|
1250
|
+
The scope of an input file generator.
|
1251
|
+
"""
|
963
1252
|
return cls(typ=ActionScopeType.INPUT_FILE_GENERATOR, file=file)
|
964
1253
|
|
965
1254
|
@classmethod
|
966
1255
|
def output_file_parser(cls, output=None):
|
1256
|
+
"""
|
1257
|
+
The scope of an output file parser.
|
1258
|
+
"""
|
967
1259
|
return cls(typ=ActionScopeType.OUTPUT_FILE_PARSER, output=output)
|
968
1260
|
|
969
1261
|
|
970
1262
|
@dataclass
|
971
1263
|
class ActionEnvironment(JSONLike):
|
1264
|
+
"""
|
1265
|
+
The environment that an action is enacted within.
|
1266
|
+
"""
|
1267
|
+
|
972
1268
|
_app_attr = "app"
|
973
1269
|
|
974
1270
|
_child_objects = (
|
@@ -978,7 +1274,9 @@ class ActionEnvironment(JSONLike):
|
|
978
1274
|
),
|
979
1275
|
)
|
980
1276
|
|
1277
|
+
#: The environment document.
|
981
1278
|
environment: Union[str, Dict[str, Any]]
|
1279
|
+
#: The scope.
|
982
1280
|
scope: Optional[app.ActionScope] = None
|
983
1281
|
|
984
1282
|
def __post_init__(self):
|
@@ -998,8 +1296,27 @@ class ActionEnvironment(JSONLike):
|
|
998
1296
|
|
999
1297
|
|
1000
1298
|
class ActionRule(JSONLike):
|
1001
|
-
"""
|
1002
|
-
|
1299
|
+
"""
|
1300
|
+
Class to represent a rule/condition that must be True if an action is to be
|
1301
|
+
included.
|
1302
|
+
|
1303
|
+
Parameters
|
1304
|
+
----------
|
1305
|
+
rule: ~hpcflow.app.Rule
|
1306
|
+
The rule to apply.
|
1307
|
+
check_exists: str
|
1308
|
+
A special rule that is enabled if this named attribute is present.
|
1309
|
+
check_missing: str
|
1310
|
+
A special rule that is enabled if this named attribute is absent.
|
1311
|
+
path: str
|
1312
|
+
Where to find the attribute to check.
|
1313
|
+
condition: dict | ConditionLike
|
1314
|
+
A more complex condition to apply.
|
1315
|
+
cast: str
|
1316
|
+
The name of a class to cast the attribute to before checking.
|
1317
|
+
doc: str
|
1318
|
+
Documentation for this rule, if any.
|
1319
|
+
"""
|
1003
1320
|
|
1004
1321
|
_child_objects = (ChildObjectSpec(name="rule", class_name="Rule"),)
|
1005
1322
|
|
@@ -1031,8 +1348,11 @@ class ActionRule(JSONLike):
|
|
1031
1348
|
f"constructor arguments."
|
1032
1349
|
)
|
1033
1350
|
|
1351
|
+
#: The rule to apply.
|
1034
1352
|
self.rule = rule
|
1353
|
+
#: The action that contains this rule.
|
1035
1354
|
self.action = None # assigned by parent action
|
1355
|
+
#: The command that is guarded by this rule.
|
1036
1356
|
self.command = None # assigned by parent command
|
1037
1357
|
|
1038
1358
|
def __eq__(self, other):
|
@@ -1044,19 +1364,85 @@ class ActionRule(JSONLike):
|
|
1044
1364
|
|
1045
1365
|
@TimeIt.decorator
|
1046
1366
|
def test(self, element_iteration: app.ElementIteration) -> bool:
|
1367
|
+
"""
|
1368
|
+
Test if this rule holds for a particular iteration.
|
1369
|
+
|
1370
|
+
Parameter
|
1371
|
+
---------
|
1372
|
+
element_iteration:
|
1373
|
+
The iteration to apply this rule to.
|
1374
|
+
"""
|
1047
1375
|
return self.rule.test(element_like=element_iteration, action=self.action)
|
1048
1376
|
|
1049
1377
|
@classmethod
|
1050
1378
|
def check_exists(cls, check_exists):
|
1379
|
+
"""
|
1380
|
+
Make an action rule that checks if a named attribute is present.
|
1381
|
+
|
1382
|
+
Parameter
|
1383
|
+
---------
|
1384
|
+
check_exists: str
|
1385
|
+
The path to the attribute to check for.
|
1386
|
+
"""
|
1051
1387
|
return cls(rule=app.Rule(check_exists=check_exists))
|
1052
1388
|
|
1053
1389
|
@classmethod
|
1054
1390
|
def check_missing(cls, check_missing):
|
1391
|
+
"""
|
1392
|
+
Make an action rule that checks if a named attribute is absent.
|
1393
|
+
|
1394
|
+
Parameter
|
1395
|
+
---------
|
1396
|
+
check_missing: str
|
1397
|
+
The path to the attribute to check for.
|
1398
|
+
"""
|
1055
1399
|
return cls(rule=app.Rule(check_missing=check_missing))
|
1056
1400
|
|
1057
1401
|
|
1058
1402
|
class Action(JSONLike):
|
1059
|
-
"""
|
1403
|
+
"""
|
1404
|
+
An atomic component of a workflow that will be enacted within an iteration
|
1405
|
+
structure.
|
1406
|
+
|
1407
|
+
Parameters
|
1408
|
+
----------
|
1409
|
+
environments: list[ActionEnvironment]
|
1410
|
+
The environments in which this action can run.
|
1411
|
+
commands: list[~hpcflow.app.Command]
|
1412
|
+
The commands to be run by this action.
|
1413
|
+
script: str
|
1414
|
+
The name of the Python script to run.
|
1415
|
+
script_data_in: str
|
1416
|
+
Information about data input to the script.
|
1417
|
+
script_data_out: str
|
1418
|
+
Information about data output from the script.
|
1419
|
+
script_data_files_use_opt: bool
|
1420
|
+
If True, script data input and output file paths will be passed to the script
|
1421
|
+
execution command line with an option like ``--input-json`` or ``--output-hdf5``
|
1422
|
+
etc. If False, the file paths will be passed on their own. For Python scripts,
|
1423
|
+
options are always passed, and this parameter is overwritten to be True,
|
1424
|
+
regardless of its initial value.
|
1425
|
+
script_exe: str
|
1426
|
+
The executable to use to run the script.
|
1427
|
+
script_pass_env_spec: bool
|
1428
|
+
Whether to pass the environment details to the script.
|
1429
|
+
abortable: bool
|
1430
|
+
Whether this action can be aborted.
|
1431
|
+
input_file_generators: list[~hpcflow.app.InputFileGenerator]
|
1432
|
+
Any applicable input file generators.
|
1433
|
+
output_file_parsers: list[~hpcflow.app.OutputFileParser]
|
1434
|
+
Any applicable output file parsers.
|
1435
|
+
input_files: list[~hpcflow.app.FileSpec]
|
1436
|
+
The input files to the action's commands.
|
1437
|
+
output_files: list[~hpcflow.app.FileSpec]
|
1438
|
+
The output files from the action's commands.
|
1439
|
+
rules: list[ActionRule]
|
1440
|
+
How to determine whether to run the action.
|
1441
|
+
save_files: list[str]
|
1442
|
+
The names of files to be explicitly saved after each step.
|
1443
|
+
clean_up: list[str]
|
1444
|
+
The names of files to be deleted after each step.
|
1445
|
+
"""
|
1060
1446
|
|
1061
1447
|
_app_attr = "app"
|
1062
1448
|
_child_objects = (
|
@@ -1136,36 +1522,45 @@ class Action(JSONLike):
|
|
1136
1522
|
save_files: Optional[List[str]] = None,
|
1137
1523
|
clean_up: Optional[List[str]] = None,
|
1138
1524
|
):
|
1139
|
-
|
1140
|
-
Parameters
|
1141
|
-
----------
|
1142
|
-
script_data_files_use_opt
|
1143
|
-
If True, script data input and output file paths will be passed to the script
|
1144
|
-
execution command line with an option like `--input-json` or `--output-hdf5`
|
1145
|
-
etc. If False, the file paths will be passed on their own. For Python scripts,
|
1146
|
-
options are always passed, and this parameter is overwritten to be True,
|
1147
|
-
regardless of its initial value.
|
1148
|
-
|
1149
|
-
"""
|
1525
|
+
#: The commands to be run by this action.
|
1150
1526
|
self.commands = commands or []
|
1527
|
+
#: The name of the Python script to run.
|
1151
1528
|
self.script = script
|
1529
|
+
#: Information about data input to the script.
|
1152
1530
|
self.script_data_in = script_data_in
|
1531
|
+
#: Information about data output from the script.
|
1153
1532
|
self.script_data_out = script_data_out
|
1533
|
+
#: If True, script data input and output file paths will be passed to the script
|
1534
|
+
#: execution command line with an option like `--input-json` or `--output-hdf5`
|
1535
|
+
#: etc. If False, the file paths will be passed on their own. For Python scripts,
|
1536
|
+
#: options are always passed, and this parameter is overwritten to be True,
|
1537
|
+
#: regardless of its initial value.
|
1154
1538
|
self.script_data_files_use_opt = (
|
1155
1539
|
script_data_files_use_opt if not self.script_is_python else True
|
1156
1540
|
)
|
1541
|
+
#: The executable to use to run the script.
|
1157
1542
|
self.script_exe = script_exe.lower() if script_exe else None
|
1543
|
+
#: Whether to pass the environment details to the script.
|
1158
1544
|
self.script_pass_env_spec = script_pass_env_spec
|
1545
|
+
#: The environments in which this action can run.
|
1159
1546
|
self.environments = environments or [
|
1160
1547
|
self.app.ActionEnvironment(environment="null_env")
|
1161
1548
|
]
|
1549
|
+
#: Whether this action can be aborted.
|
1162
1550
|
self.abortable = abortable
|
1551
|
+
#: Any applicable input file generators.
|
1163
1552
|
self.input_file_generators = input_file_generators or []
|
1553
|
+
#: Any applicable output file parsers.
|
1164
1554
|
self.output_file_parsers = output_file_parsers or []
|
1555
|
+
#: The input files to the action's commands.
|
1165
1556
|
self.input_files = self._resolve_input_files(input_files or [])
|
1557
|
+
#: The output files from the action's commands.
|
1166
1558
|
self.output_files = self._resolve_output_files(output_files or [])
|
1559
|
+
#: How to determine whether to run the action.
|
1167
1560
|
self.rules = rules or []
|
1561
|
+
#: The names of files to be explicitly saved after each step.
|
1168
1562
|
self.save_files = save_files or []
|
1563
|
+
#: The names of files to be deleted after each step.
|
1169
1564
|
self.clean_up = clean_up or []
|
1170
1565
|
|
1171
1566
|
self._task_schema = None # assigned by parent TaskSchema
|
@@ -1174,6 +1569,9 @@ class Action(JSONLike):
|
|
1174
1569
|
self._set_parent_refs()
|
1175
1570
|
|
1176
1571
|
def process_script_data_formats(self):
|
1572
|
+
"""
|
1573
|
+
Convert script data information into standard form.
|
1574
|
+
"""
|
1177
1575
|
self.script_data_in = self._process_script_data_in(self.script_data_in)
|
1178
1576
|
self.script_data_out = self._process_script_data_out(self.script_data_out)
|
1179
1577
|
|
@@ -1317,6 +1715,9 @@ class Action(JSONLike):
|
|
1317
1715
|
|
1318
1716
|
@property
|
1319
1717
|
def task_schema(self):
|
1718
|
+
"""
|
1719
|
+
The task schema that this action came from.
|
1720
|
+
"""
|
1320
1721
|
return self._task_schema
|
1321
1722
|
|
1322
1723
|
def _resolve_input_files(self, input_files):
|
@@ -1397,7 +1798,7 @@ class Action(JSONLike):
|
|
1397
1798
|
out = {"input_file_writers": writer_files, "commands": commands}
|
1398
1799
|
return out
|
1399
1800
|
|
1400
|
-
def
|
1801
|
+
def _get_resolved_action_env(
|
1401
1802
|
self,
|
1402
1803
|
relevant_scopes: Tuple[app.ActionScopeType],
|
1403
1804
|
input_file_generator: app.InputFileGenerator = None,
|
@@ -1427,7 +1828,10 @@ class Action(JSONLike):
|
|
1427
1828
|
def get_input_file_generator_action_env(
|
1428
1829
|
self, input_file_generator: app.InputFileGenerator
|
1429
1830
|
):
|
1430
|
-
|
1831
|
+
"""
|
1832
|
+
Get the actual environment to use for an input file generator.
|
1833
|
+
"""
|
1834
|
+
return self._get_resolved_action_env(
|
1431
1835
|
relevant_scopes=(
|
1432
1836
|
ActionScopeType.ANY,
|
1433
1837
|
ActionScopeType.PROCESSING,
|
@@ -1437,7 +1841,10 @@ class Action(JSONLike):
|
|
1437
1841
|
)
|
1438
1842
|
|
1439
1843
|
def get_output_file_parser_action_env(self, output_file_parser: app.OutputFileParser):
|
1440
|
-
|
1844
|
+
"""
|
1845
|
+
Get the actual environment to use for an output file parser.
|
1846
|
+
"""
|
1847
|
+
return self._get_resolved_action_env(
|
1441
1848
|
relevant_scopes=(
|
1442
1849
|
ActionScopeType.ANY,
|
1443
1850
|
ActionScopeType.PROCESSING,
|
@@ -1447,15 +1854,24 @@ class Action(JSONLike):
|
|
1447
1854
|
)
|
1448
1855
|
|
1449
1856
|
def get_commands_action_env(self):
|
1450
|
-
|
1857
|
+
"""
|
1858
|
+
Get the actual environment to use for the action commands.
|
1859
|
+
"""
|
1860
|
+
return self._get_resolved_action_env(
|
1451
1861
|
relevant_scopes=(ActionScopeType.ANY, ActionScopeType.MAIN),
|
1452
1862
|
commands=self.commands,
|
1453
1863
|
)
|
1454
1864
|
|
1455
1865
|
def get_environment_name(self) -> str:
|
1866
|
+
"""
|
1867
|
+
Get the name of the primary environment.
|
1868
|
+
"""
|
1456
1869
|
return self.get_environment_spec()["name"]
|
1457
1870
|
|
1458
1871
|
def get_environment_spec(self) -> Dict[str, Any]:
|
1872
|
+
"""
|
1873
|
+
Get the specification for the primary envionment, assuming it has been expanded.
|
1874
|
+
"""
|
1459
1875
|
if not self._from_expand:
|
1460
1876
|
raise RuntimeError(
|
1461
1877
|
f"Cannot choose a single environment from this action because it is not "
|
@@ -1464,6 +1880,9 @@ class Action(JSONLike):
|
|
1464
1880
|
return self.environments[0].environment
|
1465
1881
|
|
1466
1882
|
def get_environment(self) -> app.Environment:
|
1883
|
+
"""
|
1884
|
+
Get the primary environment.
|
1885
|
+
"""
|
1467
1886
|
return self.app.envs.get(**self.get_environment_spec())
|
1468
1887
|
|
1469
1888
|
@staticmethod
|
@@ -1487,6 +1906,9 @@ class Action(JSONLike):
|
|
1487
1906
|
def get_snippet_script_str(
|
1488
1907
|
cls, script, env_spec: Optional[Dict[str, Any]] = None
|
1489
1908
|
) -> str:
|
1909
|
+
"""
|
1910
|
+
Get the substituted script snippet path as a string.
|
1911
|
+
"""
|
1490
1912
|
if not cls.is_snippet_script(script):
|
1491
1913
|
raise ValueError(
|
1492
1914
|
f"Must be an app-data script name (e.g. "
|
@@ -1508,6 +1930,9 @@ class Action(JSONLike):
|
|
1508
1930
|
def get_snippet_script_path(
|
1509
1931
|
cls, script_path, env_spec: Optional[Dict[str, Any]] = None
|
1510
1932
|
) -> Path:
|
1933
|
+
"""
|
1934
|
+
Get the substituted script snippet path, or False if there is no snippet.
|
1935
|
+
"""
|
1511
1936
|
if not cls.is_snippet_script(script_path):
|
1512
1937
|
return False
|
1513
1938
|
|
@@ -1518,26 +1943,42 @@ class Action(JSONLike):
|
|
1518
1943
|
return Path(path)
|
1519
1944
|
|
1520
1945
|
@staticmethod
|
1521
|
-
def
|
1946
|
+
def __get_param_dump_file_stem(js_idx: int, js_act_idx: int):
|
1522
1947
|
return RunDirAppFiles.get_run_param_dump_file_prefix(js_idx, js_act_idx)
|
1523
1948
|
|
1524
1949
|
@staticmethod
|
1525
|
-
def
|
1950
|
+
def __get_param_load_file_stem(js_idx: int, js_act_idx: int):
|
1526
1951
|
return RunDirAppFiles.get_run_param_load_file_prefix(js_idx, js_act_idx)
|
1527
1952
|
|
1528
1953
|
def get_param_dump_file_path_JSON(self, js_idx: int, js_act_idx: int):
|
1529
|
-
|
1954
|
+
"""
|
1955
|
+
Get the path of the JSON dump file.
|
1956
|
+
"""
|
1957
|
+
return Path(self.__get_param_dump_file_stem(js_idx, js_act_idx) + ".json")
|
1530
1958
|
|
1531
1959
|
def get_param_dump_file_path_HDF5(self, js_idx: int, js_act_idx: int):
|
1532
|
-
|
1960
|
+
"""
|
1961
|
+
Get the path of the HDF56 dump file.
|
1962
|
+
"""
|
1963
|
+
return Path(self.__get_param_dump_file_stem(js_idx, js_act_idx) + ".h5")
|
1533
1964
|
|
1534
1965
|
def get_param_load_file_path_JSON(self, js_idx: int, js_act_idx: int):
|
1535
|
-
|
1966
|
+
"""
|
1967
|
+
Get the path of the JSON load file.
|
1968
|
+
"""
|
1969
|
+
return Path(self.__get_param_load_file_stem(js_idx, js_act_idx) + ".json")
|
1536
1970
|
|
1537
1971
|
def get_param_load_file_path_HDF5(self, js_idx: int, js_act_idx: int):
|
1538
|
-
|
1972
|
+
"""
|
1973
|
+
Get the path of the HDF5 load file.
|
1974
|
+
"""
|
1975
|
+
return Path(self.__get_param_load_file_stem(js_idx, js_act_idx) + ".h5")
|
1539
1976
|
|
1540
1977
|
def expand(self):
|
1978
|
+
"""
|
1979
|
+
Expand this action into a list of actions if necessary.
|
1980
|
+
This converts input file generators and output file parsers into their own actions.
|
1981
|
+
"""
|
1541
1982
|
if self._from_expand:
|
1542
1983
|
# already expanded
|
1543
1984
|
return [self]
|
@@ -1694,7 +2135,7 @@ class Action(JSONLike):
|
|
1694
2135
|
|
1695
2136
|
Parameters
|
1696
2137
|
----------
|
1697
|
-
sub_parameters
|
2138
|
+
sub_parameters:
|
1698
2139
|
If True, sub-parameters (i.e. dot-delimited parameter types) will be returned
|
1699
2140
|
untouched. If False (default), only return the root parameter type and
|
1700
2141
|
disregard the sub-parameter part.
|
@@ -1747,7 +2188,7 @@ class Action(JSONLike):
|
|
1747
2188
|
|
1748
2189
|
Parameters
|
1749
2190
|
----------
|
1750
|
-
sub_parameters
|
2191
|
+
sub_parameters:
|
1751
2192
|
If True, sub-parameters (i.e. dot-delimited parameter types) in command line
|
1752
2193
|
inputs will be returned untouched. If False (default), only return the root
|
1753
2194
|
parameter type and disregard the sub-parameter part.
|
@@ -1786,9 +2227,15 @@ class Action(JSONLike):
|
|
1786
2227
|
return tuple(set(params))
|
1787
2228
|
|
1788
2229
|
def get_input_file_labels(self):
|
2230
|
+
"""
|
2231
|
+
Get the labels from the input files.
|
2232
|
+
"""
|
1789
2233
|
return tuple(i.label for i in self.input_files)
|
1790
2234
|
|
1791
2235
|
def get_output_file_labels(self):
|
2236
|
+
"""
|
2237
|
+
Get the labels from the output files.
|
2238
|
+
"""
|
1792
2239
|
return tuple(i.label for i in self.output_files)
|
1793
2240
|
|
1794
2241
|
@TimeIt.decorator
|
@@ -1919,6 +2366,10 @@ class Action(JSONLike):
|
|
1919
2366
|
return scopes
|
1920
2367
|
|
1921
2368
|
def get_precise_scope(self) -> app.ActionScope:
|
2369
|
+
"""
|
2370
|
+
Get the exact scope of this action.
|
2371
|
+
The action must have been expanded prior to calling this.
|
2372
|
+
"""
|
1922
2373
|
if not self._from_expand:
|
1923
2374
|
raise RuntimeError(
|
1924
2375
|
"Precise scope cannot be unambiguously defined until the Action has been "
|
@@ -1942,6 +2393,9 @@ class Action(JSONLike):
|
|
1942
2393
|
def is_input_type_required(
|
1943
2394
|
self, typ: str, provided_files: List[app.FileSpec]
|
1944
2395
|
) -> bool:
|
2396
|
+
"""
|
2397
|
+
Determine if the given input type is required by this action.
|
2398
|
+
"""
|
1945
2399
|
# TODO: for now assume a script takes all inputs
|
1946
2400
|
if (
|
1947
2401
|
self.script
|
@@ -1966,6 +2420,9 @@ class Action(JSONLike):
|
|
1966
2420
|
if typ in (OFP.inputs or []):
|
1967
2421
|
return True
|
1968
2422
|
|
2423
|
+
# Appears to be not required
|
2424
|
+
return False
|
2425
|
+
|
1969
2426
|
@TimeIt.decorator
|
1970
2427
|
def test_rules(self, element_iter) -> Tuple[bool, List[int]]:
|
1971
2428
|
"""Test all rules against the specified element iteration."""
|