hpcflow-new2 0.2.0a50__py3-none-any.whl → 0.2.0a52__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. hpcflow/_version.py +1 -1
  2. hpcflow/sdk/__init__.py +1 -1
  3. hpcflow/sdk/api.py +1 -1
  4. hpcflow/sdk/app.py +20 -11
  5. hpcflow/sdk/cli.py +34 -59
  6. hpcflow/sdk/core/__init__.py +13 -1
  7. hpcflow/sdk/core/actions.py +235 -126
  8. hpcflow/sdk/core/command_files.py +32 -24
  9. hpcflow/sdk/core/element.py +110 -114
  10. hpcflow/sdk/core/errors.py +57 -0
  11. hpcflow/sdk/core/loop.py +18 -34
  12. hpcflow/sdk/core/parameters.py +5 -3
  13. hpcflow/sdk/core/task.py +135 -131
  14. hpcflow/sdk/core/task_schema.py +11 -4
  15. hpcflow/sdk/core/utils.py +110 -2
  16. hpcflow/sdk/core/workflow.py +964 -676
  17. hpcflow/sdk/data/template_components/environments.yaml +0 -44
  18. hpcflow/sdk/data/template_components/task_schemas.yaml +52 -10
  19. hpcflow/sdk/persistence/__init__.py +21 -33
  20. hpcflow/sdk/persistence/base.py +1340 -458
  21. hpcflow/sdk/persistence/json.py +424 -546
  22. hpcflow/sdk/persistence/pending.py +563 -0
  23. hpcflow/sdk/persistence/store_resource.py +131 -0
  24. hpcflow/sdk/persistence/utils.py +57 -0
  25. hpcflow/sdk/persistence/zarr.py +852 -841
  26. hpcflow/sdk/submission/jobscript.py +133 -112
  27. hpcflow/sdk/submission/shells/bash.py +62 -16
  28. hpcflow/sdk/submission/shells/powershell.py +87 -16
  29. hpcflow/sdk/submission/submission.py +59 -35
  30. hpcflow/tests/unit/test_element.py +4 -9
  31. hpcflow/tests/unit/test_persistence.py +218 -0
  32. hpcflow/tests/unit/test_task.py +11 -12
  33. hpcflow/tests/unit/test_utils.py +82 -0
  34. hpcflow/tests/unit/test_workflow.py +3 -1
  35. {hpcflow_new2-0.2.0a50.dist-info → hpcflow_new2-0.2.0a52.dist-info}/METADATA +3 -1
  36. {hpcflow_new2-0.2.0a50.dist-info → hpcflow_new2-0.2.0a52.dist-info}/RECORD +38 -34
  37. {hpcflow_new2-0.2.0a50.dist-info → hpcflow_new2-0.2.0a52.dist-info}/WHEEL +0 -0
  38. {hpcflow_new2-0.2.0a50.dist-info → hpcflow_new2-0.2.0a52.dist-info}/entry_points.txt +0 -0
@@ -1,6 +1,7 @@
1
1
  from __future__ import annotations
2
2
  import copy
3
3
  from dataclasses import dataclass
4
+ from datetime import datetime
4
5
  import enum
5
6
  from pathlib import Path
6
7
  import re
@@ -9,60 +10,17 @@ from textwrap import dedent
9
10
  from typing import Any, Dict, List, Optional, Tuple, Union
10
11
 
11
12
  from valida.rules import Rule
13
+ from watchdog.utils.dirsnapshot import DirectorySnapshotDiff
12
14
 
13
15
  from hpcflow.sdk import app
14
- from hpcflow.sdk.core.command_files import FileSpec, InputFileGenerator, OutputFileParser
15
- from hpcflow.sdk.core.commands import Command
16
- from hpcflow.sdk.core.environment import Environment
17
16
  from hpcflow.sdk.core.errors import MissingCompatibleActionEnvironment
18
17
  from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
18
+ from hpcflow.sdk.core.utils import JSONLikeDirSnapShot
19
19
 
20
20
 
21
21
  ACTION_SCOPE_REGEX = r"(\w*)(?:\[(.*)\])?"
22
22
 
23
23
 
24
- @dataclass(eq=True, frozen=True)
25
- class ElementID:
26
- task_insert_ID: int
27
- element_idx: int
28
-
29
- def __lt__(self, other):
30
- return tuple(self.__dict__.values()) < tuple(other.__dict__.values())
31
-
32
-
33
- @dataclass(eq=True, frozen=True)
34
- class IterationID(ElementID):
35
- """
36
- Parameters
37
- ----------
38
- iteration_idx :
39
- Index into the `element_iterations` list/array of the task. Note this is NOT the
40
- index into local list of ElementIterations belong to an Element.
41
- """
42
-
43
- iteration_idx: int
44
-
45
- def get_element_ID(self):
46
- return ElementID(
47
- task_insert_ID=self.task_insert_ID,
48
- element_idx=self.element_idx,
49
- )
50
-
51
-
52
- @dataclass(eq=True, frozen=True)
53
- class EAR_ID(IterationID):
54
- action_idx: int
55
- run_idx: int
56
- EAR_idx: int
57
-
58
- def get_iteration_ID(self):
59
- return IterationID(
60
- task_insert_ID=self.task_insert_ID,
61
- element_idx=self.element_idx,
62
- iteration_idx=self.iteration_idx,
63
- )
64
-
65
-
66
24
  class ActionScopeType(enum.Enum):
67
25
  ANY = 0
68
26
  MAIN = 1
@@ -82,22 +40,46 @@ ACTION_SCOPE_ALLOWED_KWARGS = {
82
40
 
83
41
  class EARSubmissionStatus(enum.Enum):
84
42
  PENDING = 0 # Not yet associated with a submission
85
- PREPARED = 1 # Associated with a submission that is not yet submitted
43
+ PREPARED = 1 # Associated with a prepared submission that is not yet submitted
86
44
  SUBMITTED = 2 # Submitted for execution
87
45
  RUNNING = 3 # Executing
88
46
  COMPLETE = 4 # Finished executing
47
+ SKIPPED = 5 # Not attempted due to a failure of an upstream EAR on which this depends
89
48
 
90
49
 
91
50
  class ElementActionRun:
92
51
  _app_attr = "app"
93
52
 
94
53
  def __init__(
95
- self, element_action, run_idx: int, index: int, data_idx: Dict, metadata: Dict
54
+ self,
55
+ id_: int,
56
+ is_pending: bool,
57
+ element_action,
58
+ index: int,
59
+ data_idx: Dict,
60
+ start_time: Union[datetime, None],
61
+ end_time: Union[datetime, None],
62
+ snapshot_start: Union[Dict, None],
63
+ snapshot_end: Union[Dict, None],
64
+ submission_idx: Union[int, None],
65
+ success: Union[bool, None],
66
+ skip: bool,
67
+ exit_code: Union[int, None],
68
+ metadata: Dict,
96
69
  ) -> None:
70
+ self._id = id_
71
+ self._is_pending = is_pending
97
72
  self._element_action = element_action
98
- self._run_idx = run_idx # local index of this run with the action
99
- self._index = index # task-wide EAR index
73
+ self._index = index # local index of this run with the action
100
74
  self._data_idx = data_idx
75
+ self._start_time = start_time
76
+ self._end_time = end_time
77
+ self._submission_idx = submission_idx
78
+ self._success = success
79
+ self._skip = skip
80
+ self._snapshot_start = snapshot_start
81
+ self._snapshot_end = snapshot_end
82
+ self._exit_code = exit_code
101
83
  self._metadata = metadata
102
84
 
103
85
  # assigned on first access of corresponding properties:
@@ -106,11 +88,34 @@ class ElementActionRun:
106
88
  self._resources = None
107
89
  self._input_files = None
108
90
  self._output_files = None
91
+ self._ss_start_obj = None
92
+ self._ss_end_obj = None
93
+ self._ss_diff_obj = None
94
+
95
+ def __repr__(self):
96
+ return (
97
+ f"{self.__class__.__name__}("
98
+ f"id={self.id_!r}, index={self.index!r}, "
99
+ f"element_action={self.element_action!r})"
100
+ )
101
+
102
+ @property
103
+ def id_(self) -> int:
104
+ return self._id
105
+
106
+ @property
107
+ def is_pending(self) -> bool:
108
+ return self._is_pending
109
109
 
110
110
  @property
111
111
  def element_action(self):
112
112
  return self._element_action
113
113
 
114
+ @property
115
+ def index(self):
116
+ """Run index."""
117
+ return self._index
118
+
114
119
  @property
115
120
  def action(self):
116
121
  return self.element_action.action
@@ -127,26 +132,17 @@ class ElementActionRun:
127
132
  def workflow(self):
128
133
  return self.element_iteration.workflow
129
134
 
130
- @property
131
- def run_idx(self):
132
- return self._run_idx
133
-
134
- @property
135
- def index(self):
136
- """Task-wide EAR index."""
137
- return self._index
138
-
139
- @property
140
- def EAR_ID(self):
141
- """EAR index object."""
142
- return EAR_ID(
143
- EAR_idx=self.index,
144
- task_insert_ID=self.task.insert_ID,
145
- element_idx=self.element.index,
146
- iteration_idx=self.element_iteration.index,
147
- action_idx=self.element_action.action_idx,
148
- run_idx=self.run_idx,
149
- )
135
+ # @property
136
+ # def EAR_ID(self):
137
+ # """EAR index object."""
138
+ # return EAR_ID(
139
+ # EAR_idx=self.index,
140
+ # task_insert_ID=self.task.insert_ID,
141
+ # element_idx=self.element.index,
142
+ # iteration_idx=self.element_iteration.index,
143
+ # action_idx=self.element_action.action_idx,
144
+ # run_idx=self.run_idx,
145
+ # )
150
146
 
151
147
  @property
152
148
  def data_idx(self):
@@ -156,21 +152,51 @@ class ElementActionRun:
156
152
  def metadata(self):
157
153
  return self._metadata
158
154
 
159
- @property
160
- def submission_idx(self):
161
- return self.metadata["submission_idx"]
162
-
163
155
  @property
164
156
  def start_time(self):
165
- return self.metadata["start_time"]
157
+ return self._start_time
166
158
 
167
159
  @property
168
160
  def end_time(self):
169
- return self.metadata["end_time"]
161
+ return self._end_time
162
+
163
+ @property
164
+ def submission_idx(self):
165
+ return self._submission_idx
170
166
 
171
167
  @property
172
168
  def success(self):
173
- return self.metadata["success"]
169
+ return self._success
170
+
171
+ @property
172
+ def skip(self):
173
+ return self._skip
174
+
175
+ @property
176
+ def snapshot_start(self):
177
+ if self._ss_start_obj is None and self._snapshot_start:
178
+ self._ss_start_obj = JSONLikeDirSnapShot(**self._snapshot_start)
179
+ return self._ss_start_obj
180
+
181
+ @property
182
+ def snapshot_end(self):
183
+ if self._ss_end_obj is None and self._snapshot_end:
184
+ self._ss_end_obj = JSONLikeDirSnapShot(**self._snapshot_end)
185
+ return self._ss_end_obj
186
+
187
+ @property
188
+ def dir_diff(self) -> DirectorySnapshotDiff:
189
+ """Get the changes to the EAR working directory due to the execution of this
190
+ EAR."""
191
+ if self._ss_diff_obj is None and self.snapshot_end:
192
+ self._ss_diff_obj = DirectorySnapshotDiff(
193
+ self.snapshot_start, self.snapshot_end
194
+ )
195
+ return self._ss_diff_obj
196
+
197
+ @property
198
+ def exit_code(self):
199
+ return self._exit_code
174
200
 
175
201
  @property
176
202
  def task(self):
@@ -178,10 +204,20 @@ class ElementActionRun:
178
204
 
179
205
  @property
180
206
  def submission_status(self):
181
- if self.metadata["end_time"] is not None:
207
+ """Return the submission status of this EAR.
208
+
209
+ Note: the submission status does not provide any information about whether the EAR
210
+ execution itself can be considered successful or not.
211
+
212
+ """
213
+
214
+ if self.skip:
215
+ return EARSubmissionStatus.SKIPPED
216
+
217
+ elif self.end_time is not None:
182
218
  return EARSubmissionStatus.COMPLETE
183
219
 
184
- elif self.metadata["start_time"] is not None:
220
+ elif self.start_time is not None:
185
221
  return EARSubmissionStatus.RUNNING
186
222
 
187
223
  elif self.submission_idx is not None:
@@ -205,7 +241,7 @@ class ElementActionRun:
205
241
  return self.element_iteration.get_data_idx(
206
242
  path,
207
243
  action_idx=self.element_action.action_idx,
208
- run_idx=self.run_idx,
244
+ run_idx=self.index,
209
245
  )
210
246
 
211
247
  def get_parameter_sources(
@@ -218,7 +254,7 @@ class ElementActionRun:
218
254
  return self.element_iteration.get_parameter_sources(
219
255
  path,
220
256
  action_idx=self.element_action.action_idx,
221
- run_idx=self.run_idx,
257
+ run_idx=self.index,
222
258
  typ=typ,
223
259
  as_strings=as_strings,
224
260
  use_task_index=use_task_index,
@@ -233,7 +269,7 @@ class ElementActionRun:
233
269
  return self.element_iteration.get(
234
270
  path=path,
235
271
  action_idx=self.element_action.action_idx,
236
- run_idx=self.run_idx,
272
+ run_idx=self.index,
237
273
  default=default,
238
274
  raise_on_missing=raise_on_missing,
239
275
  )
@@ -243,17 +279,15 @@ class ElementActionRun:
243
279
 
244
280
  out = []
245
281
  for src in self.get_parameter_sources(typ="EAR_output").values():
246
- src = copy.deepcopy(src)
247
- src.pop("type")
248
- _EAR_ID = EAR_ID(**src)
249
- if _EAR_ID != self.EAR_ID:
282
+ EAR_ID_i = src["EAR_ID"]
283
+ if EAR_ID_i != self.id_:
250
284
  # don't record a self dependency!
251
- out.append(_EAR_ID)
285
+ out.append(EAR_ID_i)
252
286
 
253
287
  out = sorted(out)
254
288
 
255
289
  if as_objects:
256
- out = self.workflow.get_EARs_from_indices(out)
290
+ out = self.workflow.get_EARs_from_IDs(out)
257
291
 
258
292
  return out
259
293
 
@@ -273,6 +307,25 @@ class ElementActionRun:
273
307
 
274
308
  return out
275
309
 
310
+ def get_dependent_EARs(
311
+ self, as_objects=False
312
+ ) -> List[Union[int, app.ElementActionRun]]:
313
+ """Get downstream EARs that depend on this EAR."""
314
+ deps = []
315
+ for task in self.workflow.tasks[self.task.index :]:
316
+ for elem in task.elements[:]:
317
+ for iter_ in elem.iterations:
318
+ for run in iter_.action_runs:
319
+ for dep_EAR_i in run.get_EAR_dependencies(as_objects=True):
320
+ # does dep_EAR_i belong to self?
321
+ if dep_EAR_i.id_ == self._id:
322
+ deps.append(run.id_)
323
+ deps = sorted(deps)
324
+ if as_objects:
325
+ deps = self.workflow.get_EARs_from_IDs(deps)
326
+
327
+ return deps
328
+
276
329
  @property
277
330
  def inputs(self):
278
331
  if not self._inputs:
@@ -360,7 +413,7 @@ class ElementActionRun:
360
413
  # TODO: can this return multiple files for a given FileSpec?
361
414
  if not self.action._from_expand:
362
415
  raise RuntimeError(
363
- f"Cannot get output file parser files this from EAR because the "
416
+ f"Cannot get output file parser files from this from EAR because the "
364
417
  f"associated action is not expanded, meaning multiple OFPs might exist."
365
418
  )
366
419
  out_files = {}
@@ -368,6 +421,17 @@ class ElementActionRun:
368
421
  out_files[file_spec.label] = Path(file_spec.name.value())
369
422
  return out_files
370
423
 
424
+ def get_OFP_inputs(self) -> Dict[str, Union[str, List[str]]]:
425
+ if not self.action._from_expand:
426
+ raise RuntimeError(
427
+ f"Cannot get output file parser inputs from this from EAR because the "
428
+ f"associated action is not expanded, meaning multiple OFPs might exist."
429
+ )
430
+ inputs = {}
431
+ for inp_typ in self.action.output_file_parsers[0].inputs or []:
432
+ inputs[inp_typ] = self.get(f"inputs.{inp_typ}")
433
+ return inputs
434
+
371
435
  def compose_source(self) -> str:
372
436
  """Generate the file contents of this source."""
373
437
 
@@ -382,35 +446,25 @@ class ElementActionRun:
382
446
  """\
383
447
  if __name__ == "__main__":
384
448
  import sys
385
- from {app_package_name}.api import {app_name} as app
449
+ import {app_module} as app
386
450
  app.load_config(
387
451
  config_dir=r"{cfg_dir}",
388
452
  config_invocation_key=r"{cfg_invoc_key}",
389
453
  )
390
- wk_path, sub_idx, js_idx, js_elem_idx, js_act_idx = sys.argv[1:]
454
+ wk_path, EAR_ID = sys.argv[1:]
455
+ EAR_ID = int(EAR_ID)
391
456
  wk = app.Workflow(wk_path)
392
- _, EAR = wk._from_internal_get_EAR(
393
- submission_idx=int(sub_idx),
394
- jobscript_idx=int(js_idx),
395
- JS_element_idx=int(js_elem_idx),
396
- JS_action_idx=int(js_act_idx),
397
- )
457
+ EAR = wk.get_EARs_from_IDs([EAR_ID])[0]
398
458
  inputs = EAR.get_input_values()
399
459
  outputs = {script_main_func}(**inputs)
400
460
  outputs = {{"outputs." + k: v for k, v in outputs.items()}}
401
- wk.save_parameters(
402
- values=outputs,
403
- submission_idx=int(sub_idx),
404
- jobscript_idx=int(js_idx),
405
- JS_element_idx=int(js_elem_idx),
406
- JS_action_idx=int(js_act_idx),
407
- )
461
+ for name_i, out_i in outputs.items():
462
+ wk.set_parameter_value(param_id=EAR.data_idx[name_i], value=out_i)
408
463
 
409
464
  """
410
465
  )
411
466
  main_block = main_block.format(
412
- app_package_name=self.app.package_name,
413
- app_name=self.app.name,
467
+ app_module=self.app.module,
414
468
  cfg_dir=self.app.config.config_directory,
415
469
  cfg_invoc_key=self.app.config.config_invocation_key,
416
470
  script_main_func=script_main_func,
@@ -463,6 +517,7 @@ class ElementActionRun:
463
517
  self.write_source()
464
518
 
465
519
  param_regex = r"(\<\<parameter:{}\>\>?)"
520
+ file_regex = r"(\<\<file:{}\>\>?)"
466
521
  exe_script_regex = r"\<\<(executable|script):(.*?)\>\>"
467
522
 
468
523
  command_lns = []
@@ -482,15 +537,28 @@ class ElementActionRun:
482
537
 
483
538
  # substitute input parameters in command:
484
539
  for cmd_inp in self.action.get_command_input_types():
485
- inp_val = self.get(f"inputs.{cmd_inp}")
540
+ inp_val = self.get(f"inputs.{cmd_inp}") # TODO: what if schema output?
486
541
  cmd_str = re.sub(
487
542
  pattern=param_regex.format(cmd_inp),
488
543
  repl=str(inp_val),
489
544
  string=cmd_str,
490
545
  )
491
546
 
547
+ # substitute input files in command:
548
+ for cmd_file in self.action.get_command_input_file_labels():
549
+ file_path = self.get(f"input_files.{cmd_file}") # TODO: what if out file?
550
+ # assuming we have copied this file to the EAR directory, then we just
551
+ # need the file name:
552
+ file_name = Path(file_path).name
553
+ cmd_str = re.sub(
554
+ pattern=file_regex.format(cmd_file),
555
+ repl=file_name,
556
+ string=cmd_str,
557
+ )
558
+
492
559
  out_types = command.get_output_types()
493
560
  if out_types["stdout"]:
561
+ # TODO: also map stderr/both if possible
494
562
  # assign stdout to a shell variable if required:
495
563
  param_name = f"outputs.{out_types['stdout']}"
496
564
  shell_var_name = f"parameter_{out_types['stdout']}"
@@ -499,8 +567,11 @@ class ElementActionRun:
499
567
  shell_var_name=shell_var_name,
500
568
  command=cmd_str,
501
569
  )
570
+ elif command.stdout:
571
+ cmd_str += f" 1>> {command.stdout}"
502
572
 
503
- # TODO: also map stderr/both if possible
573
+ if command.stderr:
574
+ cmd_str += f" 2>> {command.stderr}"
504
575
 
505
576
  command_lns.append(cmd_str)
506
577
 
@@ -528,6 +599,7 @@ class ElementAction:
528
599
  def __repr__(self):
529
600
  return (
530
601
  f"{self.__class__.__name__}("
602
+ f"iter_ID={self.element_iteration.id_}, "
531
603
  f"scope={self.action.get_precise_scope().to_string()!r}, "
532
604
  f"action_idx={self.action_idx}, num_runs={self.num_runs}"
533
605
  f")"
@@ -549,8 +621,16 @@ class ElementAction:
549
621
  def runs(self):
550
622
  if self._run_objs is None:
551
623
  self._run_objs = [
552
- self.app.ElementActionRun(self, run_idx=run_idx, **i)
553
- for run_idx, i in enumerate(self._runs)
624
+ self.app.ElementActionRun(
625
+ element_action=self,
626
+ index=idx,
627
+ **{
628
+ k: v
629
+ for k, v in i.items()
630
+ if k not in ("elem_iter_ID", "action_idx")
631
+ },
632
+ )
633
+ for idx, i in enumerate(self._runs)
554
634
  ]
555
635
  return self._run_objs
556
636
 
@@ -963,6 +1043,7 @@ class Action(JSONLike):
963
1043
  environments: List[app.ActionEnvironment],
964
1044
  commands: Optional[List[app.Command]] = None,
965
1045
  script: Optional[str] = None,
1046
+ abortable: Optional[bool] = False,
966
1047
  input_file_generators: Optional[List[app.InputFileGenerator]] = None,
967
1048
  output_file_parsers: Optional[List[app.OutputFileParser]] = None,
968
1049
  input_files: Optional[List[app.FileSpec]] = None,
@@ -972,6 +1053,7 @@ class Action(JSONLike):
972
1053
  self.commands = commands or []
973
1054
  self.script = script
974
1055
  self.environments = environments
1056
+ self.abortable = abortable
975
1057
  self.input_file_generators = input_file_generators or []
976
1058
  self.output_file_parsers = output_file_parsers or []
977
1059
  self.input_files = self._resolve_input_files(input_files or [])
@@ -1042,6 +1124,7 @@ class Action(JSONLike):
1042
1124
  self.commands == other.commands
1043
1125
  and self.script == other.script
1044
1126
  and self.environments == other.environments
1127
+ and self.abortable == other.abortable
1045
1128
  and self.input_file_generators == other.input_file_generators
1046
1129
  and self.output_file_parsers == other.output_file_parsers
1047
1130
  and self.rules == other.rules
@@ -1150,15 +1233,13 @@ class Action(JSONLike):
1150
1233
  inp_files = []
1151
1234
  inp_acts = []
1152
1235
  for ifg in self.input_file_generators:
1153
- cmd = (
1154
- f"<<executable:python>> <<script:{ifg.script}>> "
1155
- f"$WK_PATH $SUB_IDX $JS_IDX $JS_elem_idx $JS_act_idx"
1156
- )
1236
+ cmd = f"<<executable:python>> <<script:{ifg.script}>> $WK_PATH $EAR_ID"
1157
1237
  act_i = self.app.Action(
1158
1238
  commands=[app.Command(cmd)],
1159
1239
  input_file_generators=[ifg],
1160
1240
  environments=[self.get_input_file_generator_action_env(ifg)],
1161
1241
  rules=main_rules + [ifg.get_action_rule()],
1242
+ abortable=ifg.abortable,
1162
1243
  )
1163
1244
  act_i._task_schema = self.task_schema
1164
1245
  inp_files.append(ifg.input_file)
@@ -1168,15 +1249,13 @@ class Action(JSONLike):
1168
1249
  out_files = []
1169
1250
  out_acts = []
1170
1251
  for ofp in self.output_file_parsers:
1171
- cmd = (
1172
- f"<<executable:python>> <<script:{ofp.script}>> "
1173
- f"$WK_PATH $SUB_IDX $JS_IDX $JS_elem_idx $JS_act_idx"
1174
- )
1252
+ cmd = f"<<executable:python>> <<script:{ofp.script}>> $WK_PATH $EAR_ID"
1175
1253
  act_i = self.app.Action(
1176
1254
  commands=[app.Command(cmd)],
1177
1255
  output_file_parsers=[ofp],
1178
1256
  environments=[self.get_output_file_parser_action_env(ofp)],
1179
1257
  rules=list(self.rules),
1258
+ abortable=ofp.abortable,
1180
1259
  )
1181
1260
  act_i._task_schema = self.task_schema
1182
1261
  out_files.extend(ofp.output_files)
@@ -1187,8 +1266,7 @@ class Action(JSONLike):
1187
1266
  if self.script:
1188
1267
  commands += [
1189
1268
  self.app.Command(
1190
- f"<<executable:python>> <<script:{self.script}>> "
1191
- f"$WK_PATH $SUB_IDX $JS_IDX $JS_elem_idx $JS_act_idx"
1269
+ f"<<executable:python>> <<script:{self.script}>> $WK_PATH $EAR_ID"
1192
1270
  )
1193
1271
  ]
1194
1272
 
@@ -1196,6 +1274,7 @@ class Action(JSONLike):
1196
1274
  commands=commands,
1197
1275
  script=self.script,
1198
1276
  environments=[self.get_commands_action_env()],
1277
+ abortable=self.abortable,
1199
1278
  rules=main_rules,
1200
1279
  input_files=inp_files,
1201
1280
  output_files=out_files,
@@ -1219,6 +1298,16 @@ class Action(JSONLike):
1219
1298
  # TODO: consider stdin?
1220
1299
  return tuple(set(params))
1221
1300
 
1301
+ def get_command_input_file_labels(self) -> Tuple[str]:
1302
+ """Get input files types from commands."""
1303
+ files = []
1304
+ vars_regex = r"\<\<file:(.*?)\>\>"
1305
+ for command in self.commands:
1306
+ for val in re.findall(vars_regex, command.command):
1307
+ files.append(val)
1308
+ # TODO: consider stdin?
1309
+ return tuple(set(files))
1310
+
1222
1311
  def get_command_output_types(self) -> Tuple[str]:
1223
1312
  """Get parameter types from command stdout and stderr arguments."""
1224
1313
  params = []
@@ -1245,6 +1334,8 @@ class Action(JSONLike):
1245
1334
  params = list(self.get_command_input_types())
1246
1335
  for i in self.input_file_generators:
1247
1336
  params.extend([j.typ for j in i.inputs])
1337
+ for i in self.output_file_parsers:
1338
+ params.extend([j for j in i.inputs or []])
1248
1339
  return tuple(set(params))
1249
1340
 
1250
1341
  def get_output_types(self) -> Tuple[str]:
@@ -1270,10 +1361,20 @@ class Action(JSONLike):
1270
1361
  return tuple(i.label for i in self.output_files)
1271
1362
 
1272
1363
  def generate_data_index(
1273
- self, act_idx, EAR_idx, schema_data_idx, all_data_idx, workflow, param_source
1274
- ):
1364
+ self,
1365
+ act_idx,
1366
+ EAR_ID,
1367
+ schema_data_idx,
1368
+ all_data_idx,
1369
+ workflow,
1370
+ param_source,
1371
+ ) -> List[int]:
1275
1372
  """Generate the data index for this action of an element iteration whose overall
1276
- data index is passed."""
1373
+ data index is passed.
1374
+
1375
+ This mutates `all_data_idx`.
1376
+
1377
+ """
1277
1378
 
1278
1379
  # output keys must be processed first for this to work, since when processing an
1279
1380
  # output key, we may need to update the index of an output in a previous action's
@@ -1325,16 +1426,19 @@ class Action(JSONLike):
1325
1426
  else:
1326
1427
  # outputs
1327
1428
  k_idx = None
1328
- for (act_idx_i, EAR_idx_i), prev_data_idx in all_data_idx.items():
1429
+ for (act_idx_i, EAR_ID_i), prev_data_idx in all_data_idx.items():
1329
1430
  if key in prev_data_idx:
1330
1431
  k_idx = prev_data_idx[key]
1331
1432
 
1332
1433
  # allocate a new parameter datum for this intermediate output:
1333
1434
  param_source_i = copy.deepcopy(param_source)
1334
- param_source_i["action_idx"] = act_idx_i
1335
- param_source_i["EAR_idx"] = EAR_idx_i
1435
+ # param_source_i["action_idx"] = act_idx_i
1436
+ param_source_i["EAR_ID"] = EAR_ID_i
1336
1437
  new_k_idx = workflow._add_unset_parameter_data(param_source_i)
1438
+
1439
+ # mutate `all_data_idx`:
1337
1440
  prev_data_idx[key] = new_k_idx
1441
+
1338
1442
  if k_idx is None:
1339
1443
  # otherwise take from the schema_data_idx:
1340
1444
  k_idx = schema_data_idx[key]
@@ -1345,7 +1449,7 @@ class Action(JSONLike):
1345
1449
  sub_data_idx[key] = k_idx
1346
1450
  sub_data_idx.update(sub_param_idx)
1347
1451
 
1348
- all_data_idx[(act_idx, EAR_idx)] = sub_data_idx
1452
+ all_data_idx[(act_idx, EAR_ID)] = sub_data_idx
1349
1453
 
1350
1454
  return param_src_update
1351
1455
 
@@ -1413,3 +1517,8 @@ class Action(JSONLike):
1413
1517
  if typ in (i.typ for i in IFG.inputs):
1414
1518
  if IFG.input_file not in provided_files:
1415
1519
  return True
1520
+
1521
+ # typ is required if used in any output file parser
1522
+ for OFP in self.output_file_parsers:
1523
+ if typ in (OFP.inputs or []):
1524
+ return True