hpcflow-new2 0.2.0a50__py3-none-any.whl → 0.2.0a52__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/_version.py +1 -1
- hpcflow/sdk/__init__.py +1 -1
- hpcflow/sdk/api.py +1 -1
- hpcflow/sdk/app.py +20 -11
- hpcflow/sdk/cli.py +34 -59
- hpcflow/sdk/core/__init__.py +13 -1
- hpcflow/sdk/core/actions.py +235 -126
- hpcflow/sdk/core/command_files.py +32 -24
- hpcflow/sdk/core/element.py +110 -114
- hpcflow/sdk/core/errors.py +57 -0
- hpcflow/sdk/core/loop.py +18 -34
- hpcflow/sdk/core/parameters.py +5 -3
- hpcflow/sdk/core/task.py +135 -131
- hpcflow/sdk/core/task_schema.py +11 -4
- hpcflow/sdk/core/utils.py +110 -2
- hpcflow/sdk/core/workflow.py +964 -676
- hpcflow/sdk/data/template_components/environments.yaml +0 -44
- hpcflow/sdk/data/template_components/task_schemas.yaml +52 -10
- hpcflow/sdk/persistence/__init__.py +21 -33
- hpcflow/sdk/persistence/base.py +1340 -458
- hpcflow/sdk/persistence/json.py +424 -546
- hpcflow/sdk/persistence/pending.py +563 -0
- hpcflow/sdk/persistence/store_resource.py +131 -0
- hpcflow/sdk/persistence/utils.py +57 -0
- hpcflow/sdk/persistence/zarr.py +852 -841
- hpcflow/sdk/submission/jobscript.py +133 -112
- hpcflow/sdk/submission/shells/bash.py +62 -16
- hpcflow/sdk/submission/shells/powershell.py +87 -16
- hpcflow/sdk/submission/submission.py +59 -35
- hpcflow/tests/unit/test_element.py +4 -9
- hpcflow/tests/unit/test_persistence.py +218 -0
- hpcflow/tests/unit/test_task.py +11 -12
- hpcflow/tests/unit/test_utils.py +82 -0
- hpcflow/tests/unit/test_workflow.py +3 -1
- {hpcflow_new2-0.2.0a50.dist-info → hpcflow_new2-0.2.0a52.dist-info}/METADATA +3 -1
- {hpcflow_new2-0.2.0a50.dist-info → hpcflow_new2-0.2.0a52.dist-info}/RECORD +38 -34
- {hpcflow_new2-0.2.0a50.dist-info → hpcflow_new2-0.2.0a52.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a50.dist-info → hpcflow_new2-0.2.0a52.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/core/actions.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
import copy
|
3
3
|
from dataclasses import dataclass
|
4
|
+
from datetime import datetime
|
4
5
|
import enum
|
5
6
|
from pathlib import Path
|
6
7
|
import re
|
@@ -9,60 +10,17 @@ from textwrap import dedent
|
|
9
10
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
10
11
|
|
11
12
|
from valida.rules import Rule
|
13
|
+
from watchdog.utils.dirsnapshot import DirectorySnapshotDiff
|
12
14
|
|
13
15
|
from hpcflow.sdk import app
|
14
|
-
from hpcflow.sdk.core.command_files import FileSpec, InputFileGenerator, OutputFileParser
|
15
|
-
from hpcflow.sdk.core.commands import Command
|
16
|
-
from hpcflow.sdk.core.environment import Environment
|
17
16
|
from hpcflow.sdk.core.errors import MissingCompatibleActionEnvironment
|
18
17
|
from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
|
18
|
+
from hpcflow.sdk.core.utils import JSONLikeDirSnapShot
|
19
19
|
|
20
20
|
|
21
21
|
ACTION_SCOPE_REGEX = r"(\w*)(?:\[(.*)\])?"
|
22
22
|
|
23
23
|
|
24
|
-
@dataclass(eq=True, frozen=True)
|
25
|
-
class ElementID:
|
26
|
-
task_insert_ID: int
|
27
|
-
element_idx: int
|
28
|
-
|
29
|
-
def __lt__(self, other):
|
30
|
-
return tuple(self.__dict__.values()) < tuple(other.__dict__.values())
|
31
|
-
|
32
|
-
|
33
|
-
@dataclass(eq=True, frozen=True)
|
34
|
-
class IterationID(ElementID):
|
35
|
-
"""
|
36
|
-
Parameters
|
37
|
-
----------
|
38
|
-
iteration_idx :
|
39
|
-
Index into the `element_iterations` list/array of the task. Note this is NOT the
|
40
|
-
index into local list of ElementIterations belong to an Element.
|
41
|
-
"""
|
42
|
-
|
43
|
-
iteration_idx: int
|
44
|
-
|
45
|
-
def get_element_ID(self):
|
46
|
-
return ElementID(
|
47
|
-
task_insert_ID=self.task_insert_ID,
|
48
|
-
element_idx=self.element_idx,
|
49
|
-
)
|
50
|
-
|
51
|
-
|
52
|
-
@dataclass(eq=True, frozen=True)
|
53
|
-
class EAR_ID(IterationID):
|
54
|
-
action_idx: int
|
55
|
-
run_idx: int
|
56
|
-
EAR_idx: int
|
57
|
-
|
58
|
-
def get_iteration_ID(self):
|
59
|
-
return IterationID(
|
60
|
-
task_insert_ID=self.task_insert_ID,
|
61
|
-
element_idx=self.element_idx,
|
62
|
-
iteration_idx=self.iteration_idx,
|
63
|
-
)
|
64
|
-
|
65
|
-
|
66
24
|
class ActionScopeType(enum.Enum):
|
67
25
|
ANY = 0
|
68
26
|
MAIN = 1
|
@@ -82,22 +40,46 @@ ACTION_SCOPE_ALLOWED_KWARGS = {
|
|
82
40
|
|
83
41
|
class EARSubmissionStatus(enum.Enum):
|
84
42
|
PENDING = 0 # Not yet associated with a submission
|
85
|
-
PREPARED = 1 # Associated with a submission that is not yet submitted
|
43
|
+
PREPARED = 1 # Associated with a prepared submission that is not yet submitted
|
86
44
|
SUBMITTED = 2 # Submitted for execution
|
87
45
|
RUNNING = 3 # Executing
|
88
46
|
COMPLETE = 4 # Finished executing
|
47
|
+
SKIPPED = 5 # Not attempted due to a failure of an upstream EAR on which this depends
|
89
48
|
|
90
49
|
|
91
50
|
class ElementActionRun:
|
92
51
|
_app_attr = "app"
|
93
52
|
|
94
53
|
def __init__(
|
95
|
-
self,
|
54
|
+
self,
|
55
|
+
id_: int,
|
56
|
+
is_pending: bool,
|
57
|
+
element_action,
|
58
|
+
index: int,
|
59
|
+
data_idx: Dict,
|
60
|
+
start_time: Union[datetime, None],
|
61
|
+
end_time: Union[datetime, None],
|
62
|
+
snapshot_start: Union[Dict, None],
|
63
|
+
snapshot_end: Union[Dict, None],
|
64
|
+
submission_idx: Union[int, None],
|
65
|
+
success: Union[bool, None],
|
66
|
+
skip: bool,
|
67
|
+
exit_code: Union[int, None],
|
68
|
+
metadata: Dict,
|
96
69
|
) -> None:
|
70
|
+
self._id = id_
|
71
|
+
self._is_pending = is_pending
|
97
72
|
self._element_action = element_action
|
98
|
-
self.
|
99
|
-
self._index = index # task-wide EAR index
|
73
|
+
self._index = index # local index of this run with the action
|
100
74
|
self._data_idx = data_idx
|
75
|
+
self._start_time = start_time
|
76
|
+
self._end_time = end_time
|
77
|
+
self._submission_idx = submission_idx
|
78
|
+
self._success = success
|
79
|
+
self._skip = skip
|
80
|
+
self._snapshot_start = snapshot_start
|
81
|
+
self._snapshot_end = snapshot_end
|
82
|
+
self._exit_code = exit_code
|
101
83
|
self._metadata = metadata
|
102
84
|
|
103
85
|
# assigned on first access of corresponding properties:
|
@@ -106,11 +88,34 @@ class ElementActionRun:
|
|
106
88
|
self._resources = None
|
107
89
|
self._input_files = None
|
108
90
|
self._output_files = None
|
91
|
+
self._ss_start_obj = None
|
92
|
+
self._ss_end_obj = None
|
93
|
+
self._ss_diff_obj = None
|
94
|
+
|
95
|
+
def __repr__(self):
|
96
|
+
return (
|
97
|
+
f"{self.__class__.__name__}("
|
98
|
+
f"id={self.id_!r}, index={self.index!r}, "
|
99
|
+
f"element_action={self.element_action!r})"
|
100
|
+
)
|
101
|
+
|
102
|
+
@property
|
103
|
+
def id_(self) -> int:
|
104
|
+
return self._id
|
105
|
+
|
106
|
+
@property
|
107
|
+
def is_pending(self) -> bool:
|
108
|
+
return self._is_pending
|
109
109
|
|
110
110
|
@property
|
111
111
|
def element_action(self):
|
112
112
|
return self._element_action
|
113
113
|
|
114
|
+
@property
|
115
|
+
def index(self):
|
116
|
+
"""Run index."""
|
117
|
+
return self._index
|
118
|
+
|
114
119
|
@property
|
115
120
|
def action(self):
|
116
121
|
return self.element_action.action
|
@@ -127,26 +132,17 @@ class ElementActionRun:
|
|
127
132
|
def workflow(self):
|
128
133
|
return self.element_iteration.workflow
|
129
134
|
|
130
|
-
@property
|
131
|
-
def
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
"""EAR index object."""
|
142
|
-
return EAR_ID(
|
143
|
-
EAR_idx=self.index,
|
144
|
-
task_insert_ID=self.task.insert_ID,
|
145
|
-
element_idx=self.element.index,
|
146
|
-
iteration_idx=self.element_iteration.index,
|
147
|
-
action_idx=self.element_action.action_idx,
|
148
|
-
run_idx=self.run_idx,
|
149
|
-
)
|
135
|
+
# @property
|
136
|
+
# def EAR_ID(self):
|
137
|
+
# """EAR index object."""
|
138
|
+
# return EAR_ID(
|
139
|
+
# EAR_idx=self.index,
|
140
|
+
# task_insert_ID=self.task.insert_ID,
|
141
|
+
# element_idx=self.element.index,
|
142
|
+
# iteration_idx=self.element_iteration.index,
|
143
|
+
# action_idx=self.element_action.action_idx,
|
144
|
+
# run_idx=self.run_idx,
|
145
|
+
# )
|
150
146
|
|
151
147
|
@property
|
152
148
|
def data_idx(self):
|
@@ -156,21 +152,51 @@ class ElementActionRun:
|
|
156
152
|
def metadata(self):
|
157
153
|
return self._metadata
|
158
154
|
|
159
|
-
@property
|
160
|
-
def submission_idx(self):
|
161
|
-
return self.metadata["submission_idx"]
|
162
|
-
|
163
155
|
@property
|
164
156
|
def start_time(self):
|
165
|
-
return self.
|
157
|
+
return self._start_time
|
166
158
|
|
167
159
|
@property
|
168
160
|
def end_time(self):
|
169
|
-
return self.
|
161
|
+
return self._end_time
|
162
|
+
|
163
|
+
@property
|
164
|
+
def submission_idx(self):
|
165
|
+
return self._submission_idx
|
170
166
|
|
171
167
|
@property
|
172
168
|
def success(self):
|
173
|
-
return self.
|
169
|
+
return self._success
|
170
|
+
|
171
|
+
@property
|
172
|
+
def skip(self):
|
173
|
+
return self._skip
|
174
|
+
|
175
|
+
@property
|
176
|
+
def snapshot_start(self):
|
177
|
+
if self._ss_start_obj is None and self._snapshot_start:
|
178
|
+
self._ss_start_obj = JSONLikeDirSnapShot(**self._snapshot_start)
|
179
|
+
return self._ss_start_obj
|
180
|
+
|
181
|
+
@property
|
182
|
+
def snapshot_end(self):
|
183
|
+
if self._ss_end_obj is None and self._snapshot_end:
|
184
|
+
self._ss_end_obj = JSONLikeDirSnapShot(**self._snapshot_end)
|
185
|
+
return self._ss_end_obj
|
186
|
+
|
187
|
+
@property
|
188
|
+
def dir_diff(self) -> DirectorySnapshotDiff:
|
189
|
+
"""Get the changes to the EAR working directory due to the execution of this
|
190
|
+
EAR."""
|
191
|
+
if self._ss_diff_obj is None and self.snapshot_end:
|
192
|
+
self._ss_diff_obj = DirectorySnapshotDiff(
|
193
|
+
self.snapshot_start, self.snapshot_end
|
194
|
+
)
|
195
|
+
return self._ss_diff_obj
|
196
|
+
|
197
|
+
@property
|
198
|
+
def exit_code(self):
|
199
|
+
return self._exit_code
|
174
200
|
|
175
201
|
@property
|
176
202
|
def task(self):
|
@@ -178,10 +204,20 @@ class ElementActionRun:
|
|
178
204
|
|
179
205
|
@property
|
180
206
|
def submission_status(self):
|
181
|
-
|
207
|
+
"""Return the submission status of this EAR.
|
208
|
+
|
209
|
+
Note: the submission status does not provide any information about whether the EAR
|
210
|
+
execution itself can be considered successful or not.
|
211
|
+
|
212
|
+
"""
|
213
|
+
|
214
|
+
if self.skip:
|
215
|
+
return EARSubmissionStatus.SKIPPED
|
216
|
+
|
217
|
+
elif self.end_time is not None:
|
182
218
|
return EARSubmissionStatus.COMPLETE
|
183
219
|
|
184
|
-
elif self.
|
220
|
+
elif self.start_time is not None:
|
185
221
|
return EARSubmissionStatus.RUNNING
|
186
222
|
|
187
223
|
elif self.submission_idx is not None:
|
@@ -205,7 +241,7 @@ class ElementActionRun:
|
|
205
241
|
return self.element_iteration.get_data_idx(
|
206
242
|
path,
|
207
243
|
action_idx=self.element_action.action_idx,
|
208
|
-
run_idx=self.
|
244
|
+
run_idx=self.index,
|
209
245
|
)
|
210
246
|
|
211
247
|
def get_parameter_sources(
|
@@ -218,7 +254,7 @@ class ElementActionRun:
|
|
218
254
|
return self.element_iteration.get_parameter_sources(
|
219
255
|
path,
|
220
256
|
action_idx=self.element_action.action_idx,
|
221
|
-
run_idx=self.
|
257
|
+
run_idx=self.index,
|
222
258
|
typ=typ,
|
223
259
|
as_strings=as_strings,
|
224
260
|
use_task_index=use_task_index,
|
@@ -233,7 +269,7 @@ class ElementActionRun:
|
|
233
269
|
return self.element_iteration.get(
|
234
270
|
path=path,
|
235
271
|
action_idx=self.element_action.action_idx,
|
236
|
-
run_idx=self.
|
272
|
+
run_idx=self.index,
|
237
273
|
default=default,
|
238
274
|
raise_on_missing=raise_on_missing,
|
239
275
|
)
|
@@ -243,17 +279,15 @@ class ElementActionRun:
|
|
243
279
|
|
244
280
|
out = []
|
245
281
|
for src in self.get_parameter_sources(typ="EAR_output").values():
|
246
|
-
|
247
|
-
|
248
|
-
_EAR_ID = EAR_ID(**src)
|
249
|
-
if _EAR_ID != self.EAR_ID:
|
282
|
+
EAR_ID_i = src["EAR_ID"]
|
283
|
+
if EAR_ID_i != self.id_:
|
250
284
|
# don't record a self dependency!
|
251
|
-
out.append(
|
285
|
+
out.append(EAR_ID_i)
|
252
286
|
|
253
287
|
out = sorted(out)
|
254
288
|
|
255
289
|
if as_objects:
|
256
|
-
out = self.workflow.
|
290
|
+
out = self.workflow.get_EARs_from_IDs(out)
|
257
291
|
|
258
292
|
return out
|
259
293
|
|
@@ -273,6 +307,25 @@ class ElementActionRun:
|
|
273
307
|
|
274
308
|
return out
|
275
309
|
|
310
|
+
def get_dependent_EARs(
|
311
|
+
self, as_objects=False
|
312
|
+
) -> List[Union[int, app.ElementActionRun]]:
|
313
|
+
"""Get downstream EARs that depend on this EAR."""
|
314
|
+
deps = []
|
315
|
+
for task in self.workflow.tasks[self.task.index :]:
|
316
|
+
for elem in task.elements[:]:
|
317
|
+
for iter_ in elem.iterations:
|
318
|
+
for run in iter_.action_runs:
|
319
|
+
for dep_EAR_i in run.get_EAR_dependencies(as_objects=True):
|
320
|
+
# does dep_EAR_i belong to self?
|
321
|
+
if dep_EAR_i.id_ == self._id:
|
322
|
+
deps.append(run.id_)
|
323
|
+
deps = sorted(deps)
|
324
|
+
if as_objects:
|
325
|
+
deps = self.workflow.get_EARs_from_IDs(deps)
|
326
|
+
|
327
|
+
return deps
|
328
|
+
|
276
329
|
@property
|
277
330
|
def inputs(self):
|
278
331
|
if not self._inputs:
|
@@ -360,7 +413,7 @@ class ElementActionRun:
|
|
360
413
|
# TODO: can this return multiple files for a given FileSpec?
|
361
414
|
if not self.action._from_expand:
|
362
415
|
raise RuntimeError(
|
363
|
-
f"Cannot get output file parser files this from EAR because the "
|
416
|
+
f"Cannot get output file parser files from this from EAR because the "
|
364
417
|
f"associated action is not expanded, meaning multiple OFPs might exist."
|
365
418
|
)
|
366
419
|
out_files = {}
|
@@ -368,6 +421,17 @@ class ElementActionRun:
|
|
368
421
|
out_files[file_spec.label] = Path(file_spec.name.value())
|
369
422
|
return out_files
|
370
423
|
|
424
|
+
def get_OFP_inputs(self) -> Dict[str, Union[str, List[str]]]:
|
425
|
+
if not self.action._from_expand:
|
426
|
+
raise RuntimeError(
|
427
|
+
f"Cannot get output file parser inputs from this from EAR because the "
|
428
|
+
f"associated action is not expanded, meaning multiple OFPs might exist."
|
429
|
+
)
|
430
|
+
inputs = {}
|
431
|
+
for inp_typ in self.action.output_file_parsers[0].inputs or []:
|
432
|
+
inputs[inp_typ] = self.get(f"inputs.{inp_typ}")
|
433
|
+
return inputs
|
434
|
+
|
371
435
|
def compose_source(self) -> str:
|
372
436
|
"""Generate the file contents of this source."""
|
373
437
|
|
@@ -382,35 +446,25 @@ class ElementActionRun:
|
|
382
446
|
"""\
|
383
447
|
if __name__ == "__main__":
|
384
448
|
import sys
|
385
|
-
|
449
|
+
import {app_module} as app
|
386
450
|
app.load_config(
|
387
451
|
config_dir=r"{cfg_dir}",
|
388
452
|
config_invocation_key=r"{cfg_invoc_key}",
|
389
453
|
)
|
390
|
-
wk_path,
|
454
|
+
wk_path, EAR_ID = sys.argv[1:]
|
455
|
+
EAR_ID = int(EAR_ID)
|
391
456
|
wk = app.Workflow(wk_path)
|
392
|
-
|
393
|
-
submission_idx=int(sub_idx),
|
394
|
-
jobscript_idx=int(js_idx),
|
395
|
-
JS_element_idx=int(js_elem_idx),
|
396
|
-
JS_action_idx=int(js_act_idx),
|
397
|
-
)
|
457
|
+
EAR = wk.get_EARs_from_IDs([EAR_ID])[0]
|
398
458
|
inputs = EAR.get_input_values()
|
399
459
|
outputs = {script_main_func}(**inputs)
|
400
460
|
outputs = {{"outputs." + k: v for k, v in outputs.items()}}
|
401
|
-
|
402
|
-
|
403
|
-
submission_idx=int(sub_idx),
|
404
|
-
jobscript_idx=int(js_idx),
|
405
|
-
JS_element_idx=int(js_elem_idx),
|
406
|
-
JS_action_idx=int(js_act_idx),
|
407
|
-
)
|
461
|
+
for name_i, out_i in outputs.items():
|
462
|
+
wk.set_parameter_value(param_id=EAR.data_idx[name_i], value=out_i)
|
408
463
|
|
409
464
|
"""
|
410
465
|
)
|
411
466
|
main_block = main_block.format(
|
412
|
-
|
413
|
-
app_name=self.app.name,
|
467
|
+
app_module=self.app.module,
|
414
468
|
cfg_dir=self.app.config.config_directory,
|
415
469
|
cfg_invoc_key=self.app.config.config_invocation_key,
|
416
470
|
script_main_func=script_main_func,
|
@@ -463,6 +517,7 @@ class ElementActionRun:
|
|
463
517
|
self.write_source()
|
464
518
|
|
465
519
|
param_regex = r"(\<\<parameter:{}\>\>?)"
|
520
|
+
file_regex = r"(\<\<file:{}\>\>?)"
|
466
521
|
exe_script_regex = r"\<\<(executable|script):(.*?)\>\>"
|
467
522
|
|
468
523
|
command_lns = []
|
@@ -482,15 +537,28 @@ class ElementActionRun:
|
|
482
537
|
|
483
538
|
# substitute input parameters in command:
|
484
539
|
for cmd_inp in self.action.get_command_input_types():
|
485
|
-
inp_val = self.get(f"inputs.{cmd_inp}")
|
540
|
+
inp_val = self.get(f"inputs.{cmd_inp}") # TODO: what if schema output?
|
486
541
|
cmd_str = re.sub(
|
487
542
|
pattern=param_regex.format(cmd_inp),
|
488
543
|
repl=str(inp_val),
|
489
544
|
string=cmd_str,
|
490
545
|
)
|
491
546
|
|
547
|
+
# substitute input files in command:
|
548
|
+
for cmd_file in self.action.get_command_input_file_labels():
|
549
|
+
file_path = self.get(f"input_files.{cmd_file}") # TODO: what if out file?
|
550
|
+
# assuming we have copied this file to the EAR directory, then we just
|
551
|
+
# need the file name:
|
552
|
+
file_name = Path(file_path).name
|
553
|
+
cmd_str = re.sub(
|
554
|
+
pattern=file_regex.format(cmd_file),
|
555
|
+
repl=file_name,
|
556
|
+
string=cmd_str,
|
557
|
+
)
|
558
|
+
|
492
559
|
out_types = command.get_output_types()
|
493
560
|
if out_types["stdout"]:
|
561
|
+
# TODO: also map stderr/both if possible
|
494
562
|
# assign stdout to a shell variable if required:
|
495
563
|
param_name = f"outputs.{out_types['stdout']}"
|
496
564
|
shell_var_name = f"parameter_{out_types['stdout']}"
|
@@ -499,8 +567,11 @@ class ElementActionRun:
|
|
499
567
|
shell_var_name=shell_var_name,
|
500
568
|
command=cmd_str,
|
501
569
|
)
|
570
|
+
elif command.stdout:
|
571
|
+
cmd_str += f" 1>> {command.stdout}"
|
502
572
|
|
503
|
-
|
573
|
+
if command.stderr:
|
574
|
+
cmd_str += f" 2>> {command.stderr}"
|
504
575
|
|
505
576
|
command_lns.append(cmd_str)
|
506
577
|
|
@@ -528,6 +599,7 @@ class ElementAction:
|
|
528
599
|
def __repr__(self):
|
529
600
|
return (
|
530
601
|
f"{self.__class__.__name__}("
|
602
|
+
f"iter_ID={self.element_iteration.id_}, "
|
531
603
|
f"scope={self.action.get_precise_scope().to_string()!r}, "
|
532
604
|
f"action_idx={self.action_idx}, num_runs={self.num_runs}"
|
533
605
|
f")"
|
@@ -549,8 +621,16 @@ class ElementAction:
|
|
549
621
|
def runs(self):
|
550
622
|
if self._run_objs is None:
|
551
623
|
self._run_objs = [
|
552
|
-
self.app.ElementActionRun(
|
553
|
-
|
624
|
+
self.app.ElementActionRun(
|
625
|
+
element_action=self,
|
626
|
+
index=idx,
|
627
|
+
**{
|
628
|
+
k: v
|
629
|
+
for k, v in i.items()
|
630
|
+
if k not in ("elem_iter_ID", "action_idx")
|
631
|
+
},
|
632
|
+
)
|
633
|
+
for idx, i in enumerate(self._runs)
|
554
634
|
]
|
555
635
|
return self._run_objs
|
556
636
|
|
@@ -963,6 +1043,7 @@ class Action(JSONLike):
|
|
963
1043
|
environments: List[app.ActionEnvironment],
|
964
1044
|
commands: Optional[List[app.Command]] = None,
|
965
1045
|
script: Optional[str] = None,
|
1046
|
+
abortable: Optional[bool] = False,
|
966
1047
|
input_file_generators: Optional[List[app.InputFileGenerator]] = None,
|
967
1048
|
output_file_parsers: Optional[List[app.OutputFileParser]] = None,
|
968
1049
|
input_files: Optional[List[app.FileSpec]] = None,
|
@@ -972,6 +1053,7 @@ class Action(JSONLike):
|
|
972
1053
|
self.commands = commands or []
|
973
1054
|
self.script = script
|
974
1055
|
self.environments = environments
|
1056
|
+
self.abortable = abortable
|
975
1057
|
self.input_file_generators = input_file_generators or []
|
976
1058
|
self.output_file_parsers = output_file_parsers or []
|
977
1059
|
self.input_files = self._resolve_input_files(input_files or [])
|
@@ -1042,6 +1124,7 @@ class Action(JSONLike):
|
|
1042
1124
|
self.commands == other.commands
|
1043
1125
|
and self.script == other.script
|
1044
1126
|
and self.environments == other.environments
|
1127
|
+
and self.abortable == other.abortable
|
1045
1128
|
and self.input_file_generators == other.input_file_generators
|
1046
1129
|
and self.output_file_parsers == other.output_file_parsers
|
1047
1130
|
and self.rules == other.rules
|
@@ -1150,15 +1233,13 @@ class Action(JSONLike):
|
|
1150
1233
|
inp_files = []
|
1151
1234
|
inp_acts = []
|
1152
1235
|
for ifg in self.input_file_generators:
|
1153
|
-
cmd =
|
1154
|
-
f"<<executable:python>> <<script:{ifg.script}>> "
|
1155
|
-
f"$WK_PATH $SUB_IDX $JS_IDX $JS_elem_idx $JS_act_idx"
|
1156
|
-
)
|
1236
|
+
cmd = f"<<executable:python>> <<script:{ifg.script}>> $WK_PATH $EAR_ID"
|
1157
1237
|
act_i = self.app.Action(
|
1158
1238
|
commands=[app.Command(cmd)],
|
1159
1239
|
input_file_generators=[ifg],
|
1160
1240
|
environments=[self.get_input_file_generator_action_env(ifg)],
|
1161
1241
|
rules=main_rules + [ifg.get_action_rule()],
|
1242
|
+
abortable=ifg.abortable,
|
1162
1243
|
)
|
1163
1244
|
act_i._task_schema = self.task_schema
|
1164
1245
|
inp_files.append(ifg.input_file)
|
@@ -1168,15 +1249,13 @@ class Action(JSONLike):
|
|
1168
1249
|
out_files = []
|
1169
1250
|
out_acts = []
|
1170
1251
|
for ofp in self.output_file_parsers:
|
1171
|
-
cmd =
|
1172
|
-
f"<<executable:python>> <<script:{ofp.script}>> "
|
1173
|
-
f"$WK_PATH $SUB_IDX $JS_IDX $JS_elem_idx $JS_act_idx"
|
1174
|
-
)
|
1252
|
+
cmd = f"<<executable:python>> <<script:{ofp.script}>> $WK_PATH $EAR_ID"
|
1175
1253
|
act_i = self.app.Action(
|
1176
1254
|
commands=[app.Command(cmd)],
|
1177
1255
|
output_file_parsers=[ofp],
|
1178
1256
|
environments=[self.get_output_file_parser_action_env(ofp)],
|
1179
1257
|
rules=list(self.rules),
|
1258
|
+
abortable=ofp.abortable,
|
1180
1259
|
)
|
1181
1260
|
act_i._task_schema = self.task_schema
|
1182
1261
|
out_files.extend(ofp.output_files)
|
@@ -1187,8 +1266,7 @@ class Action(JSONLike):
|
|
1187
1266
|
if self.script:
|
1188
1267
|
commands += [
|
1189
1268
|
self.app.Command(
|
1190
|
-
f"<<executable:python>> <<script:{self.script}>> "
|
1191
|
-
f"$WK_PATH $SUB_IDX $JS_IDX $JS_elem_idx $JS_act_idx"
|
1269
|
+
f"<<executable:python>> <<script:{self.script}>> $WK_PATH $EAR_ID"
|
1192
1270
|
)
|
1193
1271
|
]
|
1194
1272
|
|
@@ -1196,6 +1274,7 @@ class Action(JSONLike):
|
|
1196
1274
|
commands=commands,
|
1197
1275
|
script=self.script,
|
1198
1276
|
environments=[self.get_commands_action_env()],
|
1277
|
+
abortable=self.abortable,
|
1199
1278
|
rules=main_rules,
|
1200
1279
|
input_files=inp_files,
|
1201
1280
|
output_files=out_files,
|
@@ -1219,6 +1298,16 @@ class Action(JSONLike):
|
|
1219
1298
|
# TODO: consider stdin?
|
1220
1299
|
return tuple(set(params))
|
1221
1300
|
|
1301
|
+
def get_command_input_file_labels(self) -> Tuple[str]:
|
1302
|
+
"""Get input files types from commands."""
|
1303
|
+
files = []
|
1304
|
+
vars_regex = r"\<\<file:(.*?)\>\>"
|
1305
|
+
for command in self.commands:
|
1306
|
+
for val in re.findall(vars_regex, command.command):
|
1307
|
+
files.append(val)
|
1308
|
+
# TODO: consider stdin?
|
1309
|
+
return tuple(set(files))
|
1310
|
+
|
1222
1311
|
def get_command_output_types(self) -> Tuple[str]:
|
1223
1312
|
"""Get parameter types from command stdout and stderr arguments."""
|
1224
1313
|
params = []
|
@@ -1245,6 +1334,8 @@ class Action(JSONLike):
|
|
1245
1334
|
params = list(self.get_command_input_types())
|
1246
1335
|
for i in self.input_file_generators:
|
1247
1336
|
params.extend([j.typ for j in i.inputs])
|
1337
|
+
for i in self.output_file_parsers:
|
1338
|
+
params.extend([j for j in i.inputs or []])
|
1248
1339
|
return tuple(set(params))
|
1249
1340
|
|
1250
1341
|
def get_output_types(self) -> Tuple[str]:
|
@@ -1270,10 +1361,20 @@ class Action(JSONLike):
|
|
1270
1361
|
return tuple(i.label for i in self.output_files)
|
1271
1362
|
|
1272
1363
|
def generate_data_index(
|
1273
|
-
self,
|
1274
|
-
|
1364
|
+
self,
|
1365
|
+
act_idx,
|
1366
|
+
EAR_ID,
|
1367
|
+
schema_data_idx,
|
1368
|
+
all_data_idx,
|
1369
|
+
workflow,
|
1370
|
+
param_source,
|
1371
|
+
) -> List[int]:
|
1275
1372
|
"""Generate the data index for this action of an element iteration whose overall
|
1276
|
-
data index is passed.
|
1373
|
+
data index is passed.
|
1374
|
+
|
1375
|
+
This mutates `all_data_idx`.
|
1376
|
+
|
1377
|
+
"""
|
1277
1378
|
|
1278
1379
|
# output keys must be processed first for this to work, since when processing an
|
1279
1380
|
# output key, we may need to update the index of an output in a previous action's
|
@@ -1325,16 +1426,19 @@ class Action(JSONLike):
|
|
1325
1426
|
else:
|
1326
1427
|
# outputs
|
1327
1428
|
k_idx = None
|
1328
|
-
for (act_idx_i,
|
1429
|
+
for (act_idx_i, EAR_ID_i), prev_data_idx in all_data_idx.items():
|
1329
1430
|
if key in prev_data_idx:
|
1330
1431
|
k_idx = prev_data_idx[key]
|
1331
1432
|
|
1332
1433
|
# allocate a new parameter datum for this intermediate output:
|
1333
1434
|
param_source_i = copy.deepcopy(param_source)
|
1334
|
-
param_source_i["action_idx"] = act_idx_i
|
1335
|
-
param_source_i["
|
1435
|
+
# param_source_i["action_idx"] = act_idx_i
|
1436
|
+
param_source_i["EAR_ID"] = EAR_ID_i
|
1336
1437
|
new_k_idx = workflow._add_unset_parameter_data(param_source_i)
|
1438
|
+
|
1439
|
+
# mutate `all_data_idx`:
|
1337
1440
|
prev_data_idx[key] = new_k_idx
|
1441
|
+
|
1338
1442
|
if k_idx is None:
|
1339
1443
|
# otherwise take from the schema_data_idx:
|
1340
1444
|
k_idx = schema_data_idx[key]
|
@@ -1345,7 +1449,7 @@ class Action(JSONLike):
|
|
1345
1449
|
sub_data_idx[key] = k_idx
|
1346
1450
|
sub_data_idx.update(sub_param_idx)
|
1347
1451
|
|
1348
|
-
all_data_idx[(act_idx,
|
1452
|
+
all_data_idx[(act_idx, EAR_ID)] = sub_data_idx
|
1349
1453
|
|
1350
1454
|
return param_src_update
|
1351
1455
|
|
@@ -1413,3 +1517,8 @@ class Action(JSONLike):
|
|
1413
1517
|
if typ in (i.typ for i in IFG.inputs):
|
1414
1518
|
if IFG.input_file not in provided_files:
|
1415
1519
|
return True
|
1520
|
+
|
1521
|
+
# typ is required if used in any output file parser
|
1522
|
+
for OFP in self.output_file_parsers:
|
1523
|
+
if typ in (OFP.inputs or []):
|
1524
|
+
return True
|