hpcflow-new2 0.2.0a50__py3-none-any.whl → 0.2.0a52__py3-none-any.whl

hpcflow/sdk/core/workflow.py

@@ -3,14 +3,27 @@ from contextlib import contextmanager
 import copy
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
+
 from pathlib import Path
-from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
-from warnings import warn
+import random
+import string
+import time
+from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union
+from fsspec.implementations.local import LocalFileSystem
+from fsspec.implementations.zip import ZipFileSystem
 
 import numpy as np
+from fsspec.core import url_to_fs
 
 from hpcflow.sdk import app
-from hpcflow.sdk.core.actions import EAR_ID
+from hpcflow.sdk.core import (
+    ALL_TEMPLATE_FORMATS,
+    DEFAULT_TEMPLATE_FORMAT,
+    ABORT_EXIT_CODE,
+)
+from hpcflow.sdk.persistence import store_cls_from_str_NEW, DEFAULT_STORE_FORMAT
+from hpcflow.sdk.persistence.base import TEMPLATE_COMP_TYPES, AnySEAR
+from hpcflow.sdk.persistence.utils import ask_pw_on_auth_exc, infer_store
 from hpcflow.sdk.submission.jobscript import (
     generate_EAR_resource_map,
     group_resource_map_into_jobscripts,
@@ -28,22 +41,10 @@ from .utils import (
     replace_items,
 )
 from hpcflow.sdk.core.errors import (
-    InvalidInputSourceTaskReference,
     LoopAlreadyExistsError,
     SubmissionFailure,
-    WorkflowBatchUpdateFailedError,
-    WorkflowNotFoundError,
     WorkflowSubmissionFailure,
 )
-from hpcflow.sdk.persistence import (
-    store_cls_from_path,
-    store_cls_from_str,
-    temporary_workflow_rename,
-    DEFAULT_STORE_FORMAT,
-)
-
-DEFAULT_TEMPLATE_FORMAT = "yaml"
-ALL_TEMPLATE_FORMATS = ("yaml", "json")
 
 
 class _DummyPersistentWorkflow:
@@ -63,7 +64,7 @@ class _DummyPersistentWorkflow:
         return self._data_ref[-1]
 
     def get_parameter_data(self, data_idx):
-        return (True, self._parameters[self._data_ref.index(data_idx)])
+        return self._parameters[self._data_ref.index(data_idx)]
 
     def make_persistent(self, workflow: app.Workflow):
         for dat_i, source_i in zip(self._parameters, self._sources):
@@ -265,32 +266,75 @@ class WorkflowTemplate(JSONLike):
         self.loops.append(loop)
 
 
-class Workflow:
-    """Class to represent a persistent {app_name} workflow."""
+def resolve_fsspec(path: PathLike, **kwargs) -> Tuple[Any, str, str]:
+    """
+    Parameters
+    ----------
+    kwargs
+        This can include a `password` key, for connections via SSH.
+
+    """
+
+    path = str(path)
+    if path.endswith(".zip"):
+        # `url_to_fs` does not seem to work for zip combos e.g. `zip::ssh://`, so we
+        # construct a `ZipFileSystem` ourselves and assume it is signified only by the
+        # file extension:
+        fs, pw = ask_pw_on_auth_exc(
+            ZipFileSystem,
+            fo=path,
+            mode="r",
+            target_options=kwargs or {},
+            add_pw_to="target_options",
+        )
+        path = ""
 
-    _app_attr = "app"
+    else:
+        (fs, path), pw = ask_pw_on_auth_exc(url_to_fs, str(path), **kwargs)
+        path = str(Path(path).as_posix())
+        if isinstance(fs, LocalFileSystem):
+            path = str(Path(path).resolve())
+
+    return fs, path, pw
 
+
+class Workflow:
+    _app_attr = "app"
     _default_ts_fmt = r"%Y-%m-%d %H:%M:%S.%f"
     _default_ts_name_fmt = r"%Y-%m-%d_%H%M%S"
+    _input_files_dir_name = "input_files"
+    _exec_dir_name = "execute"
+
+    def __init__(
+        self,
+        path: Union[str, Path],
+        store_fmt: Optional[str] = None,
+        fs_kwargs: Optional[Dict] = None,
+    ):
+        """
+        Parameters
+        ----------
 
-    def __init__(self, path: PathLike) -> None:
-        self.path = Path(path).resolve()
-        if not self.path.is_dir():
-            raise WorkflowNotFoundError(f"No workflow found at path: {self.path}")
+        path :
 
-        # assigned on first access to corresponding properties:
-        self._ts_fmt = None
-        self._ts_name_fmt = None
-        self._creation_info = None
+        """
+
+        fs_path = str(path)
+        fs, path, _ = resolve_fsspec(fs_path or "", **(fs_kwargs or {}))
+        store_fmt = store_fmt or infer_store(fs_path, fs)
+        store_cls = store_cls_from_str_NEW(store_fmt)
 
+        self.path = path
+
+        self._creation_info = None
+        self._fs_path = None
         self._template = None
         self._template_components = None
         self._tasks = None
         self._loops = None
         self._submissions = None
 
-        self._store = store_cls_from_path(self.path)(self)
-
+        self._store = store_cls(self.app, self, self.path, fs)
         self._in_batch_mode = False  # flag to track when processing batch updates
 
         # store indices of updates during batch update, so we can revert on failure:
@@ -300,54 +344,16 @@ class Workflow:
     def name(self):
         """The workflow name may be different from the template name, as it includes the
         creation date-timestamp if generated."""
-        return self.path.parts[-1]
-
-    def _get_empty_pending(self) -> Dict:
-        return {
-            "template_components": {k: [] for k in self.app._template_component_types},
-            "tasks": [],  # list of int
-            "loops": [],  # list of int
-            "submissions": [],  # list of int
-        }
-
-    def _accept_pending(self) -> None:
-        self._reset_pending()
-
-    def _reset_pending(self) -> None:
-        self._pending = self._get_empty_pending()
-
-    def _reject_pending(self) -> None:
-        """Revert pending changes to the in-memory representation of the workflow.
-
-        This deletes new tasks, new template component data, new loops, and new
-        submissions. Element additions to existing (non-pending) tasks are separately
-        rejected/accepted by the WorkflowTask object.
-
-        """
-        for task_idx in self._pending["tasks"][::-1]:
-            # iterate in reverse so the index references are correct
-            self.tasks._remove_object(task_idx)
-            self.template.tasks.pop(task_idx)
-
-        for comp_type, comp_indices in self._pending["template_components"].items():
-            for comp_idx in comp_indices[::-1]:
-                # iterate in reverse so the index references are correct
-                self.template_components[comp_type]._remove_object(comp_idx)
-
-        for loop_idx in self._pending["loops"][::-1]:
-            # iterate in reverse so the index references are correct
-            self.loops._remove_object(loop_idx)
-            self.template.loops.pop(loop_idx)
-
-        for sub_idx in self._pending["submissions"][::-1]:
-            # iterate in reverse so the index references are correct
-            self._submissions.pop(sub_idx)
-
-        self._reset_pending()
+        # TODO: this won't work for zip stores
+        return str(Path(self.fs_path).parts[-1])
 
     @property
     def store_format(self):
-        return self._store.store_name
+        return self._store._name
+
+    @property
+    def num_tasks(self) -> int:
+        return len(self.tasks)
 
     @classmethod
     def from_template(
@@ -440,7 +446,7 @@ class Workflow:
             The datetime format to use when generating the workflow name, where it
             includes a timestamp.
         """
-        template = self.app.WorkflowTemplate.from_YAML_file(YAML_path)
+        template = cls.app.WorkflowTemplate.from_YAML_file(YAML_path)
         return cls.from_template(
             template,
             path,
@@ -488,7 +494,7 @@ class Workflow:
             The datetime format to use when generating the workflow name, where it
             includes a timestamp.
         """
-        template = self.app.WorkflowTemplate.from_YAML_string(YAML_str)
+        template = cls.app.WorkflowTemplate.from_YAML_string(YAML_str)
         return cls.from_template(
             template,
             path,
@@ -713,193 +719,151 @@ class Workflow:
             ts_name_fmt,
         )
 
-    @contextmanager
-    def batch_update(self, is_workflow_creation: bool = False) -> Iterator[None]:
-        """A context manager that batches up structural changes to the workflow and
-        commits them to disk all together when the context manager exits."""
+    def _add_empty_task(
+        self,
+        task: app.Task,
+        new_index: Optional[int] = None,
+    ) -> app.WorkflowTask:
+        if new_index is None:
+            new_index = self.num_tasks
 
-        if self._in_batch_mode:
-            yield
-        else:
-            try:
-                self._in_batch_mode = True
-                yield
+        insert_ID = self.num_added_tasks
 
-            except Exception as err:
-                print("batch update exception!")
+        # make a copy with persistent schema inputs:
+        task_c, _ = task.to_persistent(self, insert_ID)
 
-                self._in_batch_mode = False
-                self._store.reject_pending()
+        # add to the WorkflowTemplate:
+        self.template._add_empty_task(task_c, new_index, insert_ID)
 
-                for task in self.tasks:
-                    task._reset_pending_elements()
+        # create and insert a new WorkflowTask:
+        self.tasks.add_object(
+            self.app.WorkflowTask.new_empty_task(self, task_c, new_index),
+            index=new_index,
+        )
 
-                for loop in self.loops:
-                    loop._reset_pending_num_added_iters()
+        # update persistent store:
+        task_js, temp_comps_js = task_c.to_json_like()
+        self._store.add_template_components(temp_comps_js)
+        self._store.add_task(new_index, task_js)
 
-                self._reject_pending()
+        # update in-memory workflow template components:
+        temp_comps = self.app.template_components_from_json_like(temp_comps_js)
+        for comp_type, comps in temp_comps.items():
+            for comp in comps:
+                comp._set_hash()
+                if comp not in self.template_components[comp_type]:
+                    idx = self.template_components[comp_type].add_object(comp)
+                    self._pending["template_components"][comp_type].append(idx)
 
-                if is_workflow_creation:
-                    # creation failed, so no need to keep the newly generated workflow:
-                    self._store.delete_no_confirm()
-                    self._store.reinstate_replaced_dir()
+        self._pending["tasks"].append(new_index)
 
-                raise err
+        return self.tasks[new_index]
 
-            else:
-                if self._store.has_pending:
-                    is_diff = self._store.is_modified_on_disk()
-                    if is_diff:
-                        raise WorkflowBatchUpdateFailedError(
-                            f"Workflow modified on disk since it was loaded!"
-                        )
+    def _add_task(self, task: app.Task, new_index: Optional[int] = None) -> None:
+        new_wk_task = self._add_empty_task(task=task, new_index=new_index)
+        new_wk_task._add_elements(element_sets=task.element_sets)  # TODO
 
-                    for task in self.tasks:
-                        task._accept_pending_elements()
+    def add_task(self, task: app.Task, new_index: Optional[int] = None) -> None:
+        with self._store.cached_load():
+            with self.batch_update():
+                self._add_task(task, new_index=new_index)
 
-                    for loop in self.loops:
-                        loop._accept_pending_num_added_iters()
+    def add_task_after(self, new_task: app.Task, task_ref: app.Task = None) -> None:
+        """Add a new task after the specified task.
 
-                    self._store.remove_replaced_dir()
-                    # TODO: handle errors in commit pending?
-                    self._store.commit_pending()
-                    self._accept_pending()
-                    self._in_batch_mode = False
+        Parameters
+        ----------
+        task_ref
+            If not given, the new task will be added at the end of the workflow.
 
-    @classmethod
-    def _write_empty_workflow(
-        cls,
-        template: app.WorkflowTemplate,
-        path: Optional[PathLike] = None,
-        name: Optional[str] = None,
-        overwrite: Optional[bool] = False,
-        store: Optional[str] = DEFAULT_STORE_FORMAT,
-        ts_fmt: Optional[str] = None,
-        ts_name_fmt: Optional[str] = None,
-    ) -> app.Workflow:
         """
+        new_index = task_ref.index + 1 if task_ref else None
+        self.add_task(new_task, new_index)
+        # TODO: add new downstream elements?
+
+    def add_task_before(self, new_task: app.Task, task_ref: app.Task = None) -> None:
+        """Add a new task before the specified task.
+
         Parameters
         ----------
-        path
-            The directory in which the workflow will be generated. The current directory
-            if not specified.
-        name
-            The name of the workflow. If specified, the workflow directory will be `path`
-            joined with `name`. If not specified the WorkflowTemplate name will be used,
-            in combination with a date-timestamp.
-        overwrite
-            If True and the workflow directory (`path` + `name`) already exists, the
-            existing directory will be overwritten.
-        store
-            The persistent store to use for this workflow.
-        ts_fmt
-            The datetime format to use for storing datetimes. Datetimes are always stored
-            in UTC (because Numpy does not store time zone info), so this should not
-            include a time zone name.
-        ts_name_fmt
-            The datetime format to use when generating the workflow name, where it
-            includes a timestamp.
-        """
+        task_ref
+            If not given, the new task will be added at the beginning of the workflow.
 
-        ts = datetime.now()
+        """
+        new_index = task_ref.index if task_ref else 0
+        self.add_task(new_task, new_index)
+        # TODO: add new downstream elements?
 
-        # store all times in UTC, since Numpy doesn't support time zone info:
-        ts_utc = ts.astimezone(tz=timezone.utc)
+    def _add_empty_loop(self, loop: app.Loop) -> app.WorkflowLoop:
+        """Add a new loop (zeroth iterations only) to the workflow."""
 
-        ts_name_fmt = ts_name_fmt or cls._default_ts_name_fmt
-        ts_fmt = ts_fmt or cls._default_ts_fmt
+        new_index = self.num_loops
 
-        path = Path(path or "").resolve()
-        name = name or f"{template.name}_{ts.strftime(ts_name_fmt)}"
-        workflow_path = path.joinpath(name)
+        # don't modify passed object:
+        loop_c = copy.deepcopy(loop)
 
-        replaced_dir = None
-        if workflow_path.exists():
-            if overwrite:
-                replaced_dir = temporary_workflow_rename(workflow_path)
-            else:
-                raise ValueError(f"Path already exists: {workflow_path}.")
+        # add to the WorkflowTemplate:
+        self.template._add_empty_loop(loop_c)
 
-        # make template-level inputs/resources think they are persistent:
-        wk_dummy = _DummyPersistentWorkflow()
-        param_src = {"type": "workflow_resources"}
-        for res_i in template.resources:
-            res_i.make_persistent(wk_dummy, param_src)
+        # create and insert a new WorkflowLoop:
+        self.loops.add_object(
+            self.app.WorkflowLoop.new_empty_loop(
+                index=new_index,
+                workflow=self,
+                template=loop_c,
+            )
+        )
+        wk_loop = self.loops[new_index]
 
-        template_js, template_sh = template.to_json_like(exclude=["tasks", "loops"])
-        template_js["tasks"] = []
-        template_js["loops"] = []
+        loop_js, _ = loop_c.to_json_like()
 
-        creation_info = {
-            "app_info": cls.app.get_info(),
-            "create_time": ts_utc.strftime(ts_fmt),
-            "ts_fmt": ts_fmt,
-            "ts_name_fmt": ts_name_fmt,
-        }
+        # all these element iterations will be initialised for the new loop:
+        iter_IDs = [
+            i.id_ for i in self.get_element_iterations_of_tasks(loop_c.task_insert_IDs)
+        ]
 
-        store_cls = store_cls_from_str(store)
-        store_cls.write_empty_workflow(
-            template_js=template_js,
-            template_components_js=template_sh,
-            workflow_path=workflow_path,
-            replaced_dir=replaced_dir,
-            creation_info=creation_info,
+        # update persistent store:
+        self._store.add_loop(
+            loop_template=loop_js,
+            iterable_parameters=wk_loop.iterable_parameters,
+            iter_IDs=iter_IDs,
         )
-        wk = cls(workflow_path)
 
-        # actually make template inputs/resources persistent, now the workflow exists:
-        wk_dummy.make_persistent(wk)
+        self._pending["loops"].append(new_index)
 
-        return wk
+        return wk_loop
 
-    @property
-    def ts_fmt(self):
-        if not self._ts_fmt:
-            self._ts_fmt = self._store.get_creation_info()["ts_fmt"]
-        return self._ts_fmt
+    def _add_loop(self, loop: app.Loop, parent_loop_indices: Dict = None) -> None:
+        new_wk_loop = self._add_empty_loop(loop)
+        if loop.num_iterations is not None:
+            # fixed number of iterations, so add remaining N > 0 iterations:
+            for _ in range(loop.num_iterations - 1):
+                new_wk_loop.add_iteration(parent_loop_indices=parent_loop_indices)
+
+    def add_loop(self, loop: app.Loop, parent_loop_indices: Dict = None) -> None:
+        """Add a loop to a subset of workflow tasks."""
+        with self._store.cached_load():
+            with self.batch_update():
+                self._add_loop(loop, parent_loop_indices)
 
     @property
-    def ts_name_fmt(self):
-        if not self._ts_name_fmt:
-            self._ts_name_fmt = self._store.get_creation_info()["ts_name_fmt"]
-        return self._ts_name_fmt
+    def fs_path(self):
+        if not self._fs_path:
+            self._fs_path = self._store.get_fs_path()
+        return self._fs_path
 
     @property
     def creation_info(self):
         if not self._creation_info:
-            with self._store.cached_load():
-                info = self._store.get_creation_info()
-                info["create_time"] = datetime.strptime(
-                    info["create_time"], info["ts_fmt"]
-                ).replace(tzinfo=timezone.utc)
-                self._creation_info = info
+            info = self._store.get_creation_info()
+            info["create_time"] = (
+                datetime.strptime(info["create_time"], info["ts_fmt"])
+                .replace(tzinfo=timezone.utc)
+                .astimezone()
+            )
+            self._creation_info = info
         return self._creation_info
 
-    @property
-    def num_tasks(self) -> int:
-        return len(self.tasks)
-
-    @property
-    def num_added_tasks(self) -> int:
-        with self._store.cached_load():
-            return self._store.get_num_added_tasks()
-
-    @property
-    def num_elements(self) -> int:
-        return sum(task.num_elements for task in self.tasks)
-
-    @property
-    def num_element_iterations(self) -> int:
-        return sum(task.num_element_iterations for task in self.tasks)
-
-    @property
-    def num_loops(self) -> int:
-        return len(self.loops)
-
-    @property
-    def num_submissions(self) -> int:
-        return len(self.submissions)
-
     @property
     def template_components(self) -> Dict:
         if self._template_components is None:
@@ -913,6 +877,11 @@ class Workflow:
         if self._template is None:
             with self._store.cached_load():
                 temp_js = self._store.get_template()
+
+                # TODO: insert_ID and id_ are the same thing:
+                for task in temp_js["tasks"]:
+                    task.pop("id_", None)
+
                 template = self.app.WorkflowTemplate.from_json_like(
                     temp_js, self.template_components
                 )
@@ -925,19 +894,18 @@ class Workflow:
     def tasks(self) -> app.WorkflowTaskList:
         if self._tasks is None:
             with self._store.cached_load():
-                tasks_meta = self._store.get_all_tasks_metadata()
+                all_tasks = self._store.get_tasks()
                 wk_tasks = []
-                for idx, i in enumerate(tasks_meta):
+                for i in all_tasks:
                     wk_task = self.app.WorkflowTask(
                         workflow=self,
-                        template=self.template.tasks[idx],
-                        index=idx,
-                        num_elements=i["num_elements"],
-                        num_element_iterations=i["num_element_iterations"],
-                        num_EARs=i["num_EARs"],
+                        template=self.template.tasks[i.index],
+                        index=i.index,
+                        element_IDs=i.element_IDs,
                     )
                     wk_tasks.append(wk_task)
                 self._tasks = self.app.WorkflowTaskList(wk_tasks)
+
         return self._tasks
 
     @property
@@ -945,12 +913,13 @@ class Workflow:
         if self._loops is None:
             with self._store.cached_load():
                 wk_loops = []
-                for idx, loop_dat in enumerate(self._store.get_loops()):
+                for idx, loop_dat in self._store.get_loops().items():
                     wk_loop = self.app.WorkflowLoop(
                         index=idx,
                         workflow=self,
                         template=self.template.loops[idx],
-                        **loop_dat,
+                        num_added_iterations=loop_dat["num_added_iterations"],
+                        iterable_parameters=loop_dat["iterable_parameters"],
                     )
                     wk_loops.append(wk_loop)
                 self._loops = self.app.WorkflowLoopList(wk_loops)
@@ -961,134 +930,466 @@ class Workflow:
         if self._submissions is None:
             with self._store.cached_load():
                 subs = []
-                for idx, sub_dat in enumerate(self._store.get_submissions()):
-                    sub_js = {"index": idx, "workflow": self, **sub_dat}
+                for idx, sub_dat in self._store.get_submissions().items():
+                    sub_js = {"index": idx, **sub_dat}
                     sub = self.app.Submission.from_json_like(sub_js)
+                    sub.workflow = self
                     subs.append(sub)
                 self._submissions = subs
         return self._submissions
 
     @property
-    def artifacts_path(self):
-        # TODO: allow customisation of artifacts path at submission and resources level
-        return self.path / "artifacts"
+    def num_added_tasks(self) -> int:
+        return self._store._get_num_total_added_tasks()
 
-    @property
-    def submissions_path(self):
-        return self.artifacts_path / "submissions"
+    def get_store_EARs(self, id_lst: Iterable[int]) -> List[AnySEAR]:
+        return self._store.get_EARs(id_lst)
 
-    @property
-    def task_artifacts_path(self):
-        return self.artifacts_path / "tasks"
+    def get_store_element_iterations(
+        self, id_lst: Iterable[int]
+    ) -> List[AnySElementIter]:
+        return self._store.get_element_iterations(id_lst)
 
-    def elements(self) -> Iterator[app.Element]:
-        for task in self.tasks:
-            for element in task.elements:
-                yield element
+    def get_store_elements(self, id_lst: Iterable[int]) -> List[AnySElement]:
+        return self._store.get_elements(id_lst)
 
-    def copy(self, path=None) -> app.Workflow:
-        """Copy the workflow to a new path and return the copied workflow."""
-        if path is None:
-            path = self.path.parent / Path(self.path.stem + "_copy" + self.path.suffix)
-        if path.exists():
-            raise ValueError(f"Path already exists: {path}.")
-        self._store.copy(path=path)
-        return self.app.Workflow(path=path)
+    def get_store_tasks(self, id_lst: Iterable[int]) -> List[AnySTask]:
+        return self._store.get_tasks_by_IDs(id_lst)
 
-    def delete(self):
-        self._store.delete()
+    def get_element_iteration_IDs_from_EAR_IDs(self, id_lst: Iterable[int]) -> List[int]:
+        return [i.elem_iter_ID for i in self.get_store_EARs(id_lst)]
 
-    def _delete_no_confirm(self):
-        self._store.delete_no_confirm()
+    def get_element_IDs_from_EAR_IDs(self, id_lst: Iterable[int]) -> List[int]:
+        iter_IDs = self.get_element_iteration_IDs_from_EAR_IDs(id_lst)
+        return [i.element_ID for i in self.get_store_element_iterations(iter_IDs)]
 
-    def rename(self, new_name: str):
-        raise NotImplementedError
+    def get_task_IDs_from_element_IDs(self, id_lst: Iterable[int]) -> List[int]:
+        return [i.task_ID for i in self.get_store_elements(id_lst)]
 
-    def _submit(
-        self,
-        ignore_errors: Optional[bool] = False,
-        JS_parallelism: Optional[bool] = None,
-        print_stdout: Optional[bool] = False,
-    ) -> Tuple[List[Exception], Dict[int, int]]:
-        """Submit outstanding EARs for execution."""
+    def get_EAR_IDs_of_tasks(self, id_lst: int) -> List[int]:
+        """Get EAR IDs belonging to multiple tasks"""
+        return [i.id_ for i in self.get_EARs_of_tasks(id_lst)]
 
-        # generate a new submission if there are no pending submissions:
-        pending = [i for i in self.submissions if i.needs_submit]
-        if not pending:
-            new_sub = self.add_submission(JS_parallelism=JS_parallelism)
-            if not new_sub:
-                raise ValueError("No pending element action runs to submit!")
-            pending = [new_sub]
+    def get_EARs_of_tasks(self, id_lst: Iterable[int]) -> List[app.ElementActionRun]:
+        """Get EARs belonging to multiple tasks"""
+        EARs = []
+        for i in id_lst:
+            task = self.tasks.get(insert_ID=i)
+            for elem in task.elements[:]:
+                for iter_ in elem.iterations:
+                    for run in iter_.action_runs:
+                        EARs.append(run)
+        return EARs
 
-        self.submissions_path.mkdir(exist_ok=True, parents=True)
-        self.task_artifacts_path.mkdir(exist_ok=True, parents=True)
+    def get_element_iterations_of_tasks(
+        self, id_lst: Iterable[int]
+    ) -> List[app.ElementIteration]:
+        """Get element iterations belonging to multiple tasks"""
+        iters = []
+        for i in id_lst:
+            task = self.tasks.get(insert_ID=i)
+            for elem in task.elements[:]:
+                for iter_i in elem.iterations:
+                    iters.append(iter_i)
+        return iters
+
+    def get_elements_from_IDs(self, id_lst: Iterable[int]) -> List[app.Element]:
+        """Return element objects from a list of IDs."""
+
+        store_elems = self._store.get_elements(id_lst)
+
+        task_IDs = [i.task_ID for i in store_elems]
+        store_tasks = self._store.get_tasks_by_IDs(task_IDs)
+
+        index_paths = []
+        for el, tk in zip(store_elems, store_tasks):
+            elem_idx = tk.element_IDs.index(el.id_)
+            index_paths.append(
+                {
+                    "elem_idx": elem_idx,
+                    "task_idx": tk.index,
+                }
+            )
 
-        # for direct execution the submission must be persistent at submit-time, because
-        # it will be read by a new instance of the app:
-        self._store.commit_pending()
+        objs = []
+        for idx_dat in index_paths:
+            task = self.tasks[idx_dat["task_idx"]]
+            elem = task.elements[idx_dat["elem_idx"]]
+            objs.append(elem)
 
-        # submit all pending submissions:
-        exceptions = []
-        submitted_js = {}
-        for sub in pending:
+        return objs
+
+    def get_element_iterations_from_IDs(
+        self, id_lst: Iterable[int]
+    ) -> List[app.ElementIteration]:
+        """Return element iteration objects from a list of IDs."""
+
+        store_iters = self._store.get_element_iterations(id_lst)
+
+        elem_IDs = [i.element_ID for i in store_iters]
+        store_elems = self._store.get_elements(elem_IDs)
+
+        task_IDs = [i.task_ID for i in store_elems]
+        store_tasks = self._store.get_tasks_by_IDs(task_IDs)
+
+        index_paths = []
+        for it, el, tk in zip(store_iters, store_elems, store_tasks):
+            iter_idx = el.iteration_IDs.index(it.id_)
+            elem_idx = tk.element_IDs.index(el.id_)
+            index_paths.append(
+                {
+                    "iter_idx": iter_idx,
+                    "elem_idx": elem_idx,
+                    "task_idx": tk.index,
+                }
+            )
+
+        objs = []
+        for idx_dat in index_paths:
+            task = self.tasks[idx_dat["task_idx"]]
+            elem = task.elements[idx_dat["elem_idx"]]
+            iter_ = elem.iterations[idx_dat["iter_idx"]]
+            objs.append(iter_)
+
+        return objs
+
+    def get_EARs_from_IDs(self, id_lst: Iterable[int]) -> List[app.ElementActionRun]:
+        """Return element action run objects from a list of IDs."""
+
+        store_EARs = self._store.get_EARs(id_lst)
+
+        elem_iter_IDs = [i.elem_iter_ID for i in store_EARs]
+        store_iters = self._store.get_element_iterations(elem_iter_IDs)
+
+        elem_IDs = [i.element_ID for i in store_iters]
+        store_elems = self._store.get_elements(elem_IDs)
+
+        task_IDs = [i.task_ID for i in store_elems]
+        store_tasks = self._store.get_tasks_by_IDs(task_IDs)
+
+        index_paths = []
+        for rn, it, el, tk in zip(store_EARs, store_iters, store_elems, store_tasks):
+            act_idx = rn.action_idx
+            run_idx = it.EAR_IDs[act_idx].index(rn.id_)
+            iter_idx = el.iteration_IDs.index(it.id_)
+            elem_idx = tk.element_IDs.index(el.id_)
+            index_paths.append(
+                {
+                    "run_idx": run_idx,
+                    "action_idx": act_idx,
+                    "iter_idx": iter_idx,
+                    "elem_idx": elem_idx,
+                    "task_idx": tk.index,
+                }
+            )
+
+        objs = []
+        for idx_dat in index_paths:
+            task = self.tasks[idx_dat["task_idx"]]
+            elem = task.elements[idx_dat["elem_idx"]]
+            iter_ = elem.iterations[idx_dat["iter_idx"]]
+            run = iter_.actions[idx_dat["action_idx"]].runs[idx_dat["run_idx"]]
+            objs.append(run)
+
+        return objs
+
+    def get_all_elements(self) -> List[app.Element]:
+        return self.get_elements_from_IDs(range(self.num_elements))
+
+    def get_all_element_iterations(self) -> List[app.ElementIteration]:
+        return self.get_element_iterations_from_IDs(range(self.num_element_iterations))
+
+    def get_all_EARs(self) -> List[app.ElementActionRun]:
+        return self.get_EARs_from_IDs(range(self.num_EARs))
+
+    @contextmanager
+    def batch_update(self, is_workflow_creation: bool = False) -> Iterator[None]:
+        """A context manager that batches up structural changes to the workflow and
+        commits them to disk all together when the context manager exits."""
+
+        if self._in_batch_mode:
+            yield
+        else:
             try:
-                sub_js_idx = sub.submit(
-                    self.task_artifacts_path,
-                    ignore_errors=ignore_errors,
-                    print_stdout=print_stdout,
+                self.app.persistence_logger.info(
+                    f"entering batch update (is_workflow_creation={is_workflow_creation!r})"
                 )
-                submitted_js[sub.index] = sub_js_idx
-            except SubmissionFailure as exc:
-                exceptions.append(exc)
+                self._in_batch_mode = True
+                yield
 
-        return exceptions, submitted_js
+            except Exception as err:
+                self.app.persistence_logger.error("batch update exception!")
+                self._in_batch_mode = False
+                self._store._pending.reset()
 
-    def submit(
-        self,
-        ignore_errors: Optional[bool] = False,
-        JS_parallelism: Optional[bool] = None,
-        print_stdout: Optional[bool] = False,
-    ) -> Dict[int, int]:
-        with self._store.cached_load():
-            with self.batch_update():
-                # commit updates before raising exception:
-                exceptions, submitted_js = self._submit(
-                    ignore_errors=ignore_errors,
-                    JS_parallelism=JS_parallelism,
-                    print_stdout=print_stdout,
+                for task in self.tasks:
+                    task._reset_pending_element_IDs()
+                    task.template._reset_pending_element_sets()
+
+                for loop in self.loops:
+                    loop._reset_pending_num_added_iters()
+
+                self._reject_pending()
+
+                if is_workflow_creation:
+                    # creation failed, so no need to keep the newly generated workflow:
+                    self._store.delete_no_confirm()
+                    self._store.reinstate_replaced_dir()
+
+                raise err
+
+            else:
+                if self._store._pending:
+                    # is_diff = self._store.is_modified_on_disk()
+                    # if is_diff:
+                    #     raise WorkflowBatchUpdateFailedError(
+                    #         f"Workflow modified on disk since it was loaded!"
+                    #     )
+
+                    for task in self.tasks:
+                        task._accept_pending_element_IDs()
+                        task.template._accept_pending_element_sets()
+
+                    for loop in self.loops:
+                        loop._accept_pending_num_added_iters()
+
+                    if is_workflow_creation:
+                        self._store.remove_replaced_dir()
+
+                    # TODO: handle errors in commit pending?
+                    self._store._pending.commit_all()
+
+                    self._accept_pending()
+                    self.app.persistence_logger.info("exiting batch update")
+                    self._in_batch_mode = False
+
+    @classmethod
+    def temporary_rename(cls, path: str, fs) -> List[str]:
+        """Rename an existing same-path workflow (directory) so we can restore it if
+        workflow creation fails.
+
+        Renaming will occur until the successfully completed. This means multiple new
+        paths may be created, where only the final path should be considered the
+        successfully renamed workflow. Other paths will be deleted."""
+
+        all_replaced = []
+
+        @cls.app.perm_error_retry()
+        def _temp_rename(path: str, fs) -> str:
+            temp_ext = "".join(random.choices(string.ascii_letters, k=10))
+            replaced = str(Path(f"{path}.{temp_ext}").as_posix())
+            cls.app.persistence_logger.debug(
+                f"temporary_rename: _temp_rename: {path!r} --> {replaced!r}."
+            )
+            all_replaced.append(replaced)
+            try:
+                fs.rename(path, replaced, recursive=True)
+            except TypeError:
+                # `SFTPFileSystem.rename` has no `recursive` argument:
+                fs.rename(path, replaced)
+            return replaced
+
+        @cls.app.perm_error_retry()
+        def _remove_path(path: str, fs) -> None:
+            cls.app.persistence_logger.debug(f"temporary_rename: _remove_path: {path!r}.")
+            while fs.exists(path):
+                fs.rm(path, recursive=True)
+                time.sleep(0.5)
+
+        _temp_rename(path, fs)
+
+        for i in all_replaced[:-1]:
+            _remove_path(i, fs)
+
+        return all_replaced[-1]
+
+    @classmethod
+    def _write_empty_workflow(
+        cls,
+        template: app.WorkflowTemplate,
+        path: Optional[PathLike] = None,
+        name: Optional[str] = None,
+        overwrite: Optional[bool] = False,
+        store: Optional[str] = DEFAULT_STORE_FORMAT,
+        ts_fmt: Optional[str] = None,
+        ts_name_fmt: Optional[str] = None,
+        fs_kwargs: Optional[Dict] = None,
+    ) -> app.Workflow:
+        """
+        Parameters
+        ----------
+        path
+            The directory in which the workflow will be generated. The current directory
+            if not specified.
+
+        """
+        ts = datetime.now()
+
+        # store all times in UTC, since NumPy doesn't support time zone info:
+        ts_utc = ts.astimezone(tz=timezone.utc)
+
+        ts_name_fmt = ts_name_fmt or cls._default_ts_name_fmt
+        ts_fmt = ts_fmt or cls._default_ts_fmt
+
+        name = name or f"{template.name}_{ts.strftime(ts_name_fmt)}"
+
+        fs_path = f"{path or '.'}/{name}"
+        fs_kwargs = fs_kwargs or {}
+        fs, path, pw = resolve_fsspec(path or "", **fs_kwargs)
+        wk_path = f"{path}/{name}"
+
+        replaced_wk = None
+        if fs.exists(wk_path):
+            cls.app.logger.debug("workflow path exists")
+            if overwrite:
+                cls.app.logger.debug("renaming existing workflow path")
+                replaced_wk = cls.temporary_rename(wk_path, fs)
+            else:
+                raise ValueError(
+                    f"Path already exists: {wk_path} on file system " f"{fs!r}."
                 )
 
-        if exceptions:
-            msg = "\n" + "\n\n".join([i.message for i in exceptions])
-            raise WorkflowSubmissionFailure(msg)
+        # make template-level inputs/resources think they are persistent:
+        wk_dummy = _DummyPersistentWorkflow()
+        param_src = {"type": "workflow_resources"}
+        for res_i in template.resources:
+            res_i.make_persistent(wk_dummy, param_src)
 
-        return submitted_js
+        template_js, template_sh = template.to_json_like(exclude=["tasks", "loops"])
+        template_js["tasks"] = []
+        template_js["loops"] = []
 
-    def add_submission(self, JS_parallelism: Optional[bool] = None) -> app.Submission:
-        new_idx = self.num_submissions
-        sub_obj = self.app.Submission(
-            index=new_idx,
-            workflow=self,
-            jobscripts=self.resolve_jobscripts(),
-            JS_parallelism=JS_parallelism,
+        creation_info = {
+            "app_info": cls.app.get_info(),
+            "create_time": ts_utc.strftime(ts_fmt),
+            "ts_fmt": ts_fmt,
+            "ts_name_fmt": ts_name_fmt,
+        }
+
+        store_cls = store_cls_from_str_NEW(store)
+        store_cls.write_empty_workflow(
+            app=cls.app,
+            template_js=template_js,
+            template_components_js=template_sh,
+            wk_path=wk_path,
+            fs=fs,
+            fs_path=fs_path,
+            replaced_wk=replaced_wk,
+            creation_info=creation_info,
         )
-        EAR_indices = sub_obj.prepare_EAR_submission_idx_update()
-        if not EAR_indices:
-            print(
-                f"There are no pending element action runs, so a new submission was not "
-                f"added."
-            )
-            return
 
-        self.set_EAR_submission_indices(sub_idx=new_idx, EAR_indices=EAR_indices)
-        sub_obj_js, _ = sub_obj.to_json_like()
-        self._submissions.append(sub_obj)
-        self._pending["submissions"].append(new_idx)
-        with self._store.cached_load():
-            with self.batch_update():
-                self._store.add_submission(sub_obj_js)
+        fs_kwargs = {"password": pw, **fs_kwargs}
+        wk = cls(fs_path, store_fmt=store, fs_kwargs=fs_kwargs)
 
-        return self.submissions[new_idx]
+        # actually make template inputs/resources persistent, now the workflow exists:
+        wk_dummy.make_persistent(wk)
+
+        return wk
+
+    def to_zip(self) -> str:
+        return self._store.to_zip()
+
+    def copy(self, path=None) -> str:
+        """Copy the workflow to a new path and return the copied workflow path."""
+        return self._store.copy(path)
+
+    def delete(self):
+        self._store.delete()
+
+    def _delete_no_confirm(self):
+        self._store.delete_no_confirm()
+
+    def get_parameters(
+        self, id_lst: Iterable[int], **kwargs: Dict
+    ) -> List[AnySParameter]:
+        return self._store.get_parameters(id_lst, **kwargs)
+
+    def get_parameter_sources(self, id_lst: Iterable[int]) -> List[Dict]:
+        return self._store.get_parameter_sources(id_lst)
+
+    def get_parameter_set_statuses(self, id_lst: Iterable[int]) -> List[bool]:
+        return self._store.get_parameter_set_statuses(id_lst)
+
+    def get_parameter(self, index: int, **kwargs: Dict) -> AnySParameter:
+        return self.get_parameters([index], **kwargs)[0]
+
+    def get_parameter_data(self, index: int, **kwargs: Dict) -> Any:
+        return (
+            self.get_parameter(index, **kwargs).data
+            or self.get_parameter(index, **kwargs).file
+        )
+
+    def get_parameter_source(self, index: int) -> Dict:
+        return self.get_parameter_sources([index])[0]
+
+    def is_parameter_set(self, index: int) -> bool:
+        return self.get_parameter_set_statuses([index])[0]
+
+    def get_all_parameters(self, **kwargs: Dict) -> List[AnySParameter]:
+        """Retrieve all store parameters."""
+        num_params = self._store._get_num_total_parameters()
+        id_lst = list(range(num_params))
+        return self._store.get_parameters(id_lst, **kwargs)
+
+    def get_all_parameter_data(self, **kwargs: Dict) -> Dict[int, Any]:
+        """Retrieve all workflow parameter data."""
+        params = self.get_all_parameters(**kwargs)
+        return {i.id_: (i.data or i.file) for i in params}
+
+    def check_parameters_exist(
+        self, id_lst: Union[int, List[int]]
+    ) -> Union[bool, List[bool]]:
+        is_multi = True
+        if isinstance(id_lst, int):
+            is_multi = False
+            id_lst = [id_lst]
+        exists = self._store.check_parameters_exist(id_lst)
+        if not is_multi:
+            exists = exists[0]
+        return exists
+
+    def _add_unset_parameter_data(self, source: Dict) -> int:
+        # TODO: use this for unset files as well
+        return self._store.add_unset_parameter(source)
+
+    def _add_parameter_data(self, data, source: Dict) -> int:
+        return self._store.add_set_parameter(data, source)
+
+    def _add_file(
+        self,
+        store_contents: bool,
+        is_input: bool,
+        source: Dict,
+        path=None,
+        contents=None,
+        filename: str = None,
+    ) -> int:
+        return self._store.add_file(
+            store_contents=store_contents,
+            is_input=is_input,
+            source=source,
+            path=path,
+            contents=contents,
+            filename=filename,
+        )
+
+    def _set_file(
+        self,
+        param_id: int,
+        store_contents: bool,
+        is_input: bool,
+        path=None,
+        contents=None,
+        filename: str = None,
+    ) -> int:
+        self._store.set_file(
+            param_id=param_id,
+            store_contents=store_contents,
+            is_input=is_input,
+            path=path,
+            contents=contents,
+            filename=filename,
+        )
 
     def get_task_unique_names(
         self, map_to_insert_ID: bool = False
@@ -1116,358 +1417,313 @@ class Workflow:
 
         return uniq_names[new_index]
 
-    def _add_empty_task(
-        self,
-        task: app.Task,
-        new_index: Optional[int] = None,
-    ) -> app.WorkflowTask:
-        if new_index is None:
-            new_index = self.num_tasks
+    def _get_empty_pending(self) -> Dict:
+        return {
+            "template_components": {k: [] for k in TEMPLATE_COMP_TYPES},
+            "tasks": [],  # list of int
+            "loops": [],  # list of int
+            "submissions": [],  # list of int
+        }
 
-        insert_ID = self.num_added_tasks
+    def _accept_pending(self) -> None:
+        self._reset_pending()
 
-        # make a copy with persistent schema inputs:
-        task_c, _ = task.to_persistent(self, insert_ID)
+    def _reset_pending(self) -> None:
+        self._pending = self._get_empty_pending()
 
-        # add to the WorkflowTemplate:
-        self.template._add_empty_task(task_c, new_index, insert_ID)
+    def _reject_pending(self) -> None:
+        """Revert pending changes to the in-memory representation of the workflow.
 
-        # create and insert a new WorkflowTask:
-        self.tasks.add_object(
-            self.app.WorkflowTask.new_empty_task(self, task_c, new_index),
-            index=new_index,
-        )
+        This deletes new tasks, new template component data, new loops, and new
+        submissions. Element additions to existing (non-pending) tasks are separately
+        rejected/accepted by the WorkflowTask object.
 
-        # update persistent store:
-        task_js, temp_comps_js = task_c.to_json_like()
-        self._store.add_template_components(temp_comps_js)
-        self._store.add_empty_task(new_index, task_js)
+        """
+        for task_idx in self._pending["tasks"][::-1]:
+            # iterate in reverse so the index references are correct
+            self.tasks._remove_object(task_idx)
+            self.template.tasks.pop(task_idx)
 
-        # update in-memory workflow template components:
-        temp_comps = self.app.template_components_from_json_like(temp_comps_js)
-        for comp_type, comps in temp_comps.items():
-            for comp in comps:
-                comp._set_hash()
-                if comp not in self.template_components[comp_type]:
-                    idx = self.template_components[comp_type].add_object(comp)
-                    self._pending["template_components"][comp_type].append(idx)
+        for comp_type, comp_indices in self._pending["template_components"].items():
+            for comp_idx in comp_indices[::-1]:
+                # iterate in reverse so the index references are correct
+                self.template_components[comp_type]._remove_object(comp_idx)
 
-        self._pending["tasks"].append(new_index)
+        for loop_idx in self._pending["loops"][::-1]:
+            # iterate in reverse so the index references are correct
+            self.loops._remove_object(loop_idx)
+            self.template.loops.pop(loop_idx)
 
-        return self.tasks[new_index]
+        for sub_idx in self._pending["submissions"][::-1]:
+            # iterate in reverse so the index references are correct
+            self._submissions.pop(sub_idx)
 
-    def _add_empty_loop(self, loop: app.Loop) -> app.WorkflowLoop:
-        """Add a new loop (zeroth iterations only) to the workflow."""
+        self._reset_pending()
 
-        new_index = self.num_loops
+    @property
+    def num_tasks(self):
+        return self._store._get_num_total_tasks()
 
-        # don't modify passed object:
-        loop_c = copy.deepcopy(loop)
+    @property
+    def num_submissions(self):
+        return self._store._get_num_total_submissions()
 
-        # add to the WorkflowTemplate:
-        self.template._add_empty_loop(loop_c)
+    @property
+    def num_elements(self):
+        return self._store._get_num_total_elements()
 
-        # create and insert a new WorkflowLoop:
-        self.loops.add_object(
-            self.app.WorkflowLoop.new_empty_loop(
-                index=new_index,
-                workflow=self,
-                template=loop_c,
-            )
-        )
-        wk_loop = self.loops[new_index]
+    @property
+    def num_element_iterations(self):
+        return self._store._get_num_total_elem_iters()
 
-        # update persistent store:
-        loop_js, _ = loop_c.to_json_like()
-        task_indices = [self.tasks.get(insert_ID=i).index for i in loop_c.task_insert_IDs]
-        self._store.add_loop(
-            task_indices=task_indices,
-            loop_js=loop_js,
-            iterable_parameters=wk_loop.iterable_parameters,
-        )
+    @property
+    def num_EARs(self):
+        return self._store._get_num_total_EARs()
 
-        self._pending["loops"].append(new_index)
+    @property
+    def num_loops(self) -> int:
+        return self._store._get_num_total_loops()
 
-        return wk_loop
+    @property
+    def artifacts_path(self):
+        # TODO: allow customisation of artifacts path at submission and resources level
+        return Path(self.path) / "artifacts"
 
-    def _add_loop(self, loop: app.Loop, parent_loop_indices: Dict = None) -> None:
-        new_wk_loop = self._add_empty_loop(loop)
-        if loop.num_iterations is not None:
-            # fixed number of iterations, so add remaining N > 0 iterations:
-            for _ in range(loop.num_iterations - 1):
-                new_wk_loop.add_iteration(parent_loop_indices=parent_loop_indices)
+    @property
+    def input_files_path(self):
+        return self.artifacts_path / self._input_files_dir_name
 
-    def add_loop(self, loop: app.Loop, parent_loop_indices: Dict = None) -> None:
-        """Add a loop to a subset of workflow tasks."""
+    @property
+    def submissions_path(self):
+        return self.artifacts_path / "submissions"
+
+    @property
+    def task_artifacts_path(self):
+        return self.artifacts_path / "tasks"
+
+    @property
+    def execution_path(self):
+        return Path(self.path) / self._exec_dir_name
+
+    def get_task_elements(self, task: app.Task, selection: slice) -> List[app.Element]:
+        return [
+            self.app.Element(task=task, **{k: v for k, v in i.items() if k != "task_ID"})
+            for i in self._store.get_task_elements(task.insert_ID, selection)
+        ]
+
+    def set_EAR_submission_index(self, EAR_ID: int, sub_idx: int) -> None:
+        """Set the submission index of an EAR."""
         with self._store.cached_load():
             with self.batch_update():
-                self._add_loop(loop, parent_loop_indices)
-
-    def _add_task(self, task: app.Task, new_index: Optional[int] = None) -> None:
-        new_wk_task = self._add_empty_task(task=task, new_index=new_index)
-        new_wk_task._add_elements(element_sets=task.element_sets)
+                self._store.set_EAR_submission_index(EAR_ID, sub_idx)
 
-    def add_task(self, task: app.Task, new_index: Optional[int] = None) -> None:
+    def set_EAR_start(self, EAR_ID: int) -> None:
+        """Set the start time on an EAR."""
+        self.app.logger.debug(f"Setting start for EAR ID {EAR_ID!r}")
         with self._store.cached_load():
             with self.batch_update():
-                self._add_task(task, new_index=new_index)
+                self._store.set_EAR_start(EAR_ID)
 
-    def add_task_after(self, new_task: app.Task, task_ref: app.Task = None) -> None:
-        """Add a new task after the specified task.
+    def set_EAR_end(self, EAR_ID: int, exit_code: int) -> None:
+        """Set the end time and exit code on an EAR.
 
-        Parameters
-        ----------
-        task_ref
-            If not given, the new task will be added at the end of the workflow.
+        If the exit code is non-zero, also set all downstream dependent EARs to be
+        skipped. Also save any generated input/output files.
 
         """
-        new_index = task_ref.index + 1 if task_ref else None
-        self.add_task(new_task, new_index)
-        # TODO: add new downstream elements?
-
-    def add_task_before(self, new_task: app.Task, task_ref: app.Task = None) -> None:
-        """Add a new task before the specified task.
+        self.app.logger.debug(
+            f"Setting end for EAR ID {EAR_ID!r} with exit code {exit_code!r}."
+        )
+        with self._store.cached_load():
+            EAR = self.get_EARs_from_IDs([EAR_ID])[0]
+            with self.batch_update():
+                success = exit_code == 0  # TODO  more sophisticated success heuristics
+                if EAR.action.abortable and exit_code == ABORT_EXIT_CODE:
+                    # the point of aborting an EAR is to continue with the workflow:
+                    success = True
+                self._store.set_EAR_end(EAR_ID, exit_code, success)
+
+                for IFG_i in EAR.action.input_file_generators:
+                    inp_file = IFG_i.input_file
+                    self.app.logger.debug(
+                        f"Saving EAR input file: {inp_file.label} for EAR ID {EAR_ID!r}."
+                    )
+                    param_id = EAR.data_idx[f"input_files.{inp_file.label}"]
+                    self._set_file(
+                        param_id=param_id,
+                        store_contents=True,  # TODO: make optional according to IFG
+                        is_input=False,
+                        path=Path(inp_file.value()).resolve(),
+                    )
 
-        Parameters
-        ----------
-        task_ref
-            If not given, the new task will be added at the beginning of the workflow.
+                for OFP_i in EAR.action.output_file_parsers:
+                    for out_file_j in OFP_i.output_files:
+                        if out_file_j.label not in OFP_i.save_files:
+                            continue
+                        self.app.logger.debug(
+                            f"Saving EAR output file: {out_file_j.label} for EAR ID "
+                            f"{EAR_ID!r}."
+                        )
+                        param_id = EAR.data_idx[f"output_files.{out_file_j.label}"]
+                        self._set_file(
+                            param_id=param_id,
+                            store_contents=True,  # TODO: make optional according to OFP
+                            is_input=False,
+                            path=Path(out_file_j.value()).resolve(),
+                        )
 
-        """
-        new_index = task_ref.index if task_ref else 0
-        self.add_task(new_task, new_index)
-        # TODO: add new downstream elements?
+                if exit_code != 0:
+                    for EAR_dep_ID in EAR.get_dependent_EARs(as_objects=False):
+                        # TODO: this needs to be recursive?
+                        self.app.logger.debug(
+                            f"Setting EAR ID {EAR_dep_ID!r} to skip because it depends on"
+                            f" EAR ID {EAR_ID!r}, which exited with a non-zero exit code:"
+                            f" {exit_code!r}."
+                        )
+                        self._store.set_EAR_skip(EAR_dep_ID)
 
-    def get_parameter_data(self, index: int) -> Tuple[bool, Any]:
-        return self._store.get_parameter_data(index)
+    def set_EAR_skip(self, EAR_ID: int) -> None:
+        """Record that an EAR is to be skipped due to an upstream failure."""
+        with self._store.cached_load():
+            with self.batch_update():
+                self._store.set_EAR_skip(EAR_ID)
 
-    def get_parameter_source(self, index: int) -> Dict:
-        return self._store.get_parameter_source(index)
+    def get_EAR_skipped(self, EAR_ID: int) -> None:
+        """Check if an EAR is to be skipped."""
+        with self._store.cached_load():
+            return self._store.get_EAR_skipped(EAR_ID)
 
-    def get_all_parameter_data(self) -> Dict[int, Any]:
-        return self._store.get_all_parameter_data()
+    def set_parameter_value(self, param_id: int, value: Any) -> None:
+        with self._store.cached_load():
+            with self.batch_update():
+                self._store.set_parameter_value(param_id, value)
 
-    def is_parameter_set(self, index: int) -> bool:
-        return self._store.is_parameter_set(index)
+    def elements(self) -> Iterator[app.Element]:
+        for task in self.tasks:
+            for element in task.elements[:]:
+                yield element
 
-    def check_parameters_exist(
-        self, indices: Union[int, List[int]]
-    ) -> Union[bool, List[bool]]:
-        return self._store.check_parameters_exist(indices)
+    def get_iteration_task_pathway(self):
+        pathway = []
+        for task in self.tasks:
+            loop_idx = {}
+            pathway.append((task.insert_ID, loop_idx))
 
-    def _add_unset_parameter_data(self, source: Dict) -> int:
-        return self._store.add_unset_parameter_data(source)
+        for loop in self.loops:  # TODO: order by depth (inner loops first?)
+            task_subset = loop.task_insert_IDs
+            subset_idx = [idx for idx, i in enumerate(pathway) if i[0] in task_subset]
+            looped_pathway = []
+            for iter_i in range(loop.num_added_iterations):
+                for j in subset_idx:
+                    item_j = copy.deepcopy(pathway[j])
+                    item_j[1][loop.name] = iter_i
+                    looped_pathway.append(item_j)
 
-    def _add_parameter_data(self, data, source: Dict) -> int:
-        return self._store.add_parameter_data(data, source)
+            # replaced pathway `sub_idx` items with `looped_pathway` items:
+            pathway = replace_items(
+                pathway, subset_idx[0], subset_idx[-1] + 1, looped_pathway
+            )
 
-    def _resolve_input_source_task_reference(
-        self, input_source: app.InputSource, new_task_name: str
-    ) -> None:
-        """Normalise the input source task reference and convert a source to a local type
-        if required."""
+        return pathway
 
-        # TODO: test thoroughly!
+    def _submit(
+        self,
+        ignore_errors: Optional[bool] = False,
+        JS_parallelism: Optional[bool] = None,
+        print_stdout: Optional[bool] = False,
+    ) -> Tuple[List[Exception], Dict[int, int]]:
+        """Submit outstanding EARs for execution."""
 
-        if isinstance(input_source.task_ref, str):
-            if input_source.task_ref == new_task_name:
-                if input_source.task_source_type is self.app.TaskSourceType.OUTPUT:
-                    raise InvalidInputSourceTaskReference(
-                        f"Input source {input_source.to_string()!r} cannot refer to the "
-                        f"outputs of its own task!"
-                    )
-                else:
-                    warn(
-                        f"Changing input source {input_source.to_string()!r} to a local "
-                        f"type, since the input source task reference refers to its own "
-                        f"task."
-                    )
-                    # TODO: add an InputSource source_type setter to reset
-                    # task_ref/source_type?
-                    input_source.source_type = self.app.InputSourceType.LOCAL
-                    input_source.task_ref = None
-                    input_source.task_source_type = None
-            else:
-                try:
-                    uniq_names_cur = self.get_task_unique_names(map_to_insert_ID=True)
-                    input_source.task_ref = uniq_names_cur[input_source.task_ref]
-                except KeyError:
-                    raise InvalidInputSourceTaskReference(
-                        f"Input source {input_source.to_string()!r} refers to a missing "
-                        f"or inaccessible task: {input_source.task_ref!r}."
-                    )
+        # generate a new submission if there are no pending submissions:
+        pending = [i for i in self.submissions if i.needs_submit]
+        if not pending:
+            new_sub = self._add_submission(JS_parallelism=JS_parallelism)
+            if not new_sub:
+                raise ValueError("No pending element action runs to submit!")
+            pending = [new_sub]
 
-    def get_task_elements(self, task: app.Task, selection: slice) -> List[app.Element]:
-        return [
-            self.app.Element(task=task, **i)
-            for i in self._store.get_task_elements(task.index, task.insert_ID, selection)
-        ]
+        self.submissions_path.mkdir(exist_ok=True, parents=True)
+        self.execution_path.mkdir(exist_ok=True, parents=True)
+        self.task_artifacts_path.mkdir(exist_ok=True, parents=True)
 
-    def get_task_elements_islice(
-        self, task: app.Task, selection: slice
-    ) -> Iterator[app.Element]:
-        for i in self._store.get_task_elements_islice(
-            task.index, task.insert_ID, selection
-        ):
-            yield self.app.Element(task=task, **i)
-
-    def get_EARs_from_IDs(self, indices: List[EAR_ID]) -> List[app.ElementActionRun]:
-        """Return element action run objects from a list of five-tuples, representing the
-        task insert ID, element index, iteration index, action index, and run index,
-        respectively.
-        """
-        objs = []
-        for _EAR_ID in indices:
-            task = self.tasks.get(insert_ID=_EAR_ID.task_insert_ID)
-            elem_iters = task.elements[_EAR_ID.element_idx].iterations
-            for i in elem_iters:
-                if i.index == _EAR_ID.iteration_idx:
-                    iter_i = i
-                    break
-            EAR_i = iter_i.actions[_EAR_ID.action_idx].runs[_EAR_ID.run_idx]
-            objs.append(EAR_i)
-        return objs
+        # for direct execution the submission must be persistent at submit-time, because
+        # it will be read by a new instance of the app:
+        self._store._pending.commit_all()
 
-    def get_element_iterations_from_IDs(
-        self, indices: List[IterationID]
-    ) -> List[app.ElementIteration]:
-        """Return element iteration objects from a list of three-tuples, representing the
-        task insert ID, element index, and iteration index, respectively.
-        """
-        objs = []
-        for iter_idx in indices:
-            iter_i = (
-                self.tasks.get(insert_ID=iter_idx.task_insert_ID)
-                .elements[iter_idx.element_idx]
-                .iterations[iter_idx.iteration_idx]
-            )
-            objs.append(iter_i)
-        return objs
+        # submit all pending submissions:
+        exceptions = []
+        submitted_js = {}
+        for sub in pending:
+            try:
+                sub_js_idx = sub.submit(
+                    ignore_errors=ignore_errors,
+                    print_stdout=print_stdout,
+                )
+                submitted_js[sub.index] = sub_js_idx
+            except SubmissionFailure as exc:
+                exceptions.append(exc)
 
-    def get_elements_from_IDs(self, indices: List[ElementID]) -> List[app.Element]:
-        """Return element objects from a list of two-tuples, representing the task insert
-        ID, and element index, respectively."""
-        return [
-            self.tasks.get(insert_ID=idx.task_insert_ID).elements[idx.element_idx]
-            for idx in indices
-        ]
+        return exceptions, submitted_js
 
-    def set_EAR_submission_indices(
+    def submit(
         self,
-        sub_idx: int,
-        EAR_indices: Tuple[int, int, int, int],
-    ) -> None:
-        """Set the submission index on an EAR."""
+        ignore_errors: Optional[bool] = False,
+        JS_parallelism: Optional[bool] = None,
+        print_stdout: Optional[bool] = False,
+    ) -> Dict[int, int]:
         with self._store.cached_load():
             with self.batch_update():
-                self._store.set_EAR_submission_indices(sub_idx, EAR_indices)
+                # commit updates before raising exception:
+                exceptions, submitted_js = self._submit(
+                    ignore_errors=ignore_errors,
+                    JS_parallelism=JS_parallelism,
+                    print_stdout=print_stdout,
+                )
 
-    def set_EAR_start(
-        self,
-        submission_idx: int,
-        jobscript_idx: int,
-        JS_element_idx: int,
-        JS_action_idx: int,
-    ) -> None:
-        """Set the start time on an EAR."""
-        with self._store.cached_load():
-            with self.batch_update():
-                jobscript = self.submissions[submission_idx].jobscripts[jobscript_idx]
-                (t_iD, _, i_idx, a_idx, r_idx, _) = jobscript.get_EAR_ID_array()[
-                    JS_action_idx, JS_element_idx
-                ].item()
-                self._store.set_EAR_start(t_iD, i_idx, a_idx, r_idx)
+        if exceptions:
+            msg = "\n" + "\n\n".join([i.message for i in exceptions])
+            raise WorkflowSubmissionFailure(msg)
 
-    def set_EAR_end(
-        self,
-        submission_idx: int,
-        jobscript_idx: int,
-        JS_element_idx: int,
-        JS_action_idx: int,
-    ) -> None:
-        """Set the end time on an EAR."""
-        with self._store.cached_load():
-            with self.batch_update():
-                jobscript = self.submissions[submission_idx].jobscripts[jobscript_idx]
-                (t_iD, _, i_idx, a_idx, r_idx, _) = jobscript.get_EAR_ID_array()[
-                    JS_action_idx, JS_element_idx
-                ].item()
-                self._store.set_EAR_end(t_iD, i_idx, a_idx, r_idx)
+        return submitted_js
 
-    def _from_internal_get_EAR(
-        self,
-        submission_idx: int,
-        jobscript_idx: int,
-        JS_element_idx: int,
-        JS_action_idx: int,
-    ):
+    def add_submission(self, JS_parallelism: Optional[bool] = None) -> app.Submission:
         with self._store.cached_load():
-            jobscript = self.submissions[submission_idx].jobscripts[jobscript_idx]
-            id_args = jobscript.get_EAR_ID_array()[JS_action_idx, JS_element_idx].item()
-            EAR_id = EAR_ID(*id_args)
-            EAR = self.get_EARs_from_IDs([EAR_id])[0]
-
-        return jobscript, EAR
+            with self.batch_update():
+                return self._add_submission(JS_parallelism)
 
-    def write_commands(
-        self,
-        submission_idx: int,
-        jobscript_idx: int,
-        JS_element_idx: int,
-        JS_action_idx: int,
-    ) -> None:
-        """Write run-time commands for a given EAR."""
-        with self._store.cached_load():
-            jobscript, EAR = self._from_internal_get_EAR(
-                submission_idx, jobscript_idx, JS_element_idx, JS_action_idx
+    def _add_submission(self, JS_parallelism: Optional[bool] = None) -> app.Submission:
+        new_idx = self.num_submissions
+        _ = self.submissions  # TODO: just to ensure `submissions` is loaded
+        sub_obj = self.app.Submission(
+            index=new_idx,
+            workflow=self,
+            jobscripts=self.resolve_jobscripts(),
+            JS_parallelism=JS_parallelism,
+        )
+        all_EAR_ID = [i for js in sub_obj.jobscripts for i in js.EAR_ID.flatten()]
+        # EAR_indices = sub_obj.prepare_EAR_submission_idx_update()
+        if not all_EAR_ID:
+            print(
+                f"There are no pending element action runs, so a new submission was not "
+                f"added."
             )
-            commands, shell_vars = EAR.compose_commands(jobscript)
-            for param_name, shell_var_name in shell_vars:
-                commands += jobscript.shell.format_save_parameter(
-                    workflow_app_alias=jobscript.workflow_app_alias,
-                    param_name=param_name,
-                    shell_var_name=shell_var_name,
-                )
-            commands = jobscript.shell.wrap_in_subshell(commands)
-            cmd_file_name = jobscript.get_commands_file_name(JS_action_idx)
-            with Path(cmd_file_name).open("wt", newline="\n") as fp:
-                # (assuming we have CD'd correctly to the element run directory)
-                fp.write(commands)
+            return
 
-    def save_parameter(
-        self,
-        name,
-        value,
-        submission_idx: int,
-        jobscript_idx: int,
-        JS_element_idx: int,
-        JS_action_idx: int,
-    ):
         with self._store.cached_load():
             with self.batch_update():
-                _, EAR = self._from_internal_get_EAR(
-                    submission_idx, jobscript_idx, JS_element_idx, JS_action_idx
-                )
-                data_idx = EAR.data_idx[name]
-                self._store.set_parameter(data_idx, value)
+                for i in all_EAR_ID:
+                    self._store.set_EAR_submission_index(EAR_ID=i, sub_idx=new_idx)
 
-    def save_parameters(
-        self,
-        values: Dict,
-        submission_idx: int,
-        jobscript_idx: int,
-        JS_element_idx: int,
-        JS_action_idx: int,
-    ):
-        """Save multiple parameters to a given EAR."""
+        # self.set_EAR_submission_indices(sub_idx=new_idx, EAR_indices=EAR_indices)
+
+        sub_obj_js, _ = sub_obj.to_json_like()
+        self._submissions.append(sub_obj)
+        self._pending["submissions"].append(new_idx)
         with self._store.cached_load():
             with self.batch_update():
-                _, EAR = self._from_internal_get_EAR(
-                    submission_idx, jobscript_idx, JS_element_idx, JS_action_idx
-                )
-                for name, value in values.items():
-                    data_idx = EAR.data_idx[name]
-                    self._store.set_parameter(data_idx, value)
+                self._store.add_submission(new_idx, sub_obj_js)
+
+        return self.submissions[new_idx]
 
     def resolve_jobscripts(self) -> List[app.Jobscript]:
         js, element_deps = self._resolve_singular_jobscripts()
@@ -1492,6 +1748,8 @@ class Workflow:
         -------
         submission_jobscripts
         all_element_deps
+            For a given jobscript index, for a given jobscript element index within that
+            jobscript, this is a list of EAR IDs dependencies of that element.
 
         """
 
@@ -1520,8 +1778,8 @@ class Workflow:
                     len(task_actions),
                     len(task_elements[task.insert_ID]),
                 )
-                EAR_idx_arr = np.empty(EAR_idx_arr_shape, dtype=np.int32)
-                EAR_idx_arr[:] = -1
+                EAR_ID_arr = np.empty(EAR_idx_arr_shape, dtype=np.int32)
+                EAR_ID_arr[:] = -1
 
                 new_js_idx = len(submission_jobscripts)
 
@@ -1530,8 +1788,8 @@ class Workflow:
                     "task_loop_idx": [loop_idx_i],
                     "task_actions": task_actions,  # map jobscript actions to task actions
                     "task_elements": task_elements,  # map jobscript elements to task elements
-                    "EARs": {},  # keys are (task insert ID, elem_idx, EAR_idx)
-                    "EAR_idx": EAR_idx_arr,
+                    # "EARs": {},  # keys are (task insert ID, elem_idx, EAR_idx)
+                    "EAR_ID": EAR_ID_arr,
                     "resources": res[js_dat["resources"]],
                     "resource_hash": res_hash[js_dat["resources"]],
                     "dependencies": {},
@@ -1540,43 +1798,51 @@ class Workflow:
                     js_elem_idx = task_elements[task.insert_ID].index((elem_idx))
                     all_EAR_IDs = []
                     for act_idx in act_indices:
-                        EAR_idx, run_idx, iter_idx = (
-                            i.item() for i in EAR_map[act_idx, elem_idx]
-                        )
+                        EAR_ID_i = EAR_map[act_idx, elem_idx].item()
+                        # EAR_idx, run_idx, iter_idx = (
+                        #     i.item() for i in EAR_map[act_idx, elem_idx]
+                        # )
                         # construct EAR_ID object so we can retrieve the EAR objects and
                         # so their dependencies:
-                        EAR_id = EAR_ID(
-                            task_insert_ID=task.insert_ID,
-                            element_idx=elem_idx,
-                            iteration_idx=iter_idx,
-                            action_idx=act_idx,
-                            run_idx=run_idx,
-                            EAR_idx=EAR_idx,
-                        )
-                        all_EAR_IDs.append(EAR_id)
-                        js_i["EARs"][(task.insert_ID, elem_idx, EAR_idx)] = (
-                            iter_idx,
-                            act_idx,
-                            run_idx,
-                        )
+                        # EAR_id = EAR_ID(
+                        #     task_insert_ID=task.insert_ID,
+                        #     element_idx=elem_idx,
+                        #     iteration_idx=iter_idx,
+                        #     action_idx=act_idx,
+                        #     run_idx=run_idx,
+                        #     EAR_idx=EAR_idx,
+                        # )
+                        all_EAR_IDs.append(EAR_ID_i)
+                        # js_i["EARs"][(task.insert_ID, elem_idx, EAR_idx)] = (
+                        #     iter_idx,
+                        #     act_idx,
+                        #     run_idx,
+                        # )
 
                         js_act_idx = task_actions.index([task.insert_ID, act_idx, 0])
-                        js_i["EAR_idx"][js_act_idx][js_elem_idx] = EAR_idx
+                        # js_i["EAR_idx"][js_act_idx][js_elem_idx] = EAR_idx
+                        js_i["EAR_ID"][js_act_idx][js_elem_idx] = EAR_ID_i
 
                     # get indices of EARs that this element depends on:
                     EAR_objs = self.get_EARs_from_IDs(all_EAR_IDs)
                     EAR_deps = [i.get_EAR_dependencies() for i in EAR_objs]
                     EAR_deps_flat = [j for i in EAR_deps for j in i]
 
+                    # print(f"{EAR_deps=}")
+                    # print(f"{EAR_deps_flat=}")
+
                     # represent EAR dependencies of this jobscripts using the same key
                     # format as in the "EARs" dict, to allow for quick lookup when
                     # resolving dependencies between jobscripts; also, no need to include
                     # EAR dependencies that are in this jobscript:
+                    # EAR_deps_EAR_idx = [
+                    #     (i.task_insert_ID, i.element_idx, i.EAR_idx)
+                    #     for i in EAR_deps_flat
+                    #     if (i.task_insert_ID, i.element_idx, i.EAR_idx)
+                    #     not in js_i["EARs"]
+                    # ]
                     EAR_deps_EAR_idx = [
-                        (i.task_insert_ID, i.element_idx, i.EAR_idx)
-                        for i in EAR_deps_flat
-                        if (i.task_insert_ID, i.element_idx, i.EAR_idx)
-                        not in js_i["EARs"]
+                        i for i in EAR_deps_flat if i not in js_i["EAR_ID"]
                     ]
                     if EAR_deps_EAR_idx:
                         if new_js_idx not in all_element_deps:
@@ -1588,43 +1854,65 @@ class Workflow:
 
         return submission_jobscripts, all_element_deps
 
-    def get_iteration_task_pathway(self):
-        pathway = []
-        for task in self.tasks:
-            loop_idx = {}
-            pathway.append((task.insert_ID, loop_idx))
-
-        for loop in self.loops:  # TODO: order by depth (inner loops first?)
-            task_subset = loop.task_insert_IDs
-            subset_idx = [idx for idx, i in enumerate(pathway) if i[0] in task_subset]
-            looped_pathway = []
-            for iter_i in range(loop.num_added_iterations):
-                for j in subset_idx:
-                    item_j = copy.deepcopy(pathway[j])
-                    item_j[1][loop.name] = iter_i
-                    looped_pathway.append(item_j)
-
-            # replaced pathway `sub_idx` items with `looped_pathway` items:
-            pathway = replace_items(
-                pathway, subset_idx[0], subset_idx[-1] + 1, looped_pathway
-            )
+    def write_commands(
+        self,
+        submission_idx: int,
+        jobscript_idx: int,
+        JS_action_idx: int,
+        EAR_ID: int,
+    ) -> None:
+        """Write run-time commands for a given EAR."""
+        with self._store.cached_load():
+            jobscript = self.submissions[submission_idx].jobscripts[jobscript_idx]
+            EAR = self.get_EARs_from_IDs([EAR_ID])[0]
+            commands, shell_vars = EAR.compose_commands(jobscript)
+            for param_name, shell_var_name in shell_vars:
+                commands += jobscript.shell.format_save_parameter(
+                    workflow_app_alias=jobscript.workflow_app_alias,
+                    param_name=param_name,
+                    shell_var_name=shell_var_name,
+                    EAR_ID=EAR_ID,
+                )
+            commands = jobscript.shell.wrap_in_subshell(commands, EAR.action.abortable)
+            cmd_file_name = jobscript.get_commands_file_name(JS_action_idx)
+            with Path(cmd_file_name).open("wt", newline="\n") as fp:
+                # (assuming we have CD'd correctly to the element run directory)
+                fp.write(commands)
 
-        return pathway
+    def save_parameter(
+        self,
+        name: str,
+        value: Any,
+        EAR_ID: int,
+    ):
+        with self._store.cached_load():
+            with self.batch_update():
+                EAR = self.get_EARs_from_IDs([EAR_ID])[0]
+                param_id = EAR.data_idx[name]
+                self.set_parameter_value(param_id, value)
 
     def show_all_EAR_statuses(self):
         print(
             f"{'task':8s} {'element':8s} {'iteration':8s} {'action':8s} "
-            f"{'run':8s} {'status':8s}"
+            f"{'run':8s} {'sub.':8s} {'exitcode':8s} {'success':8s} {'skip':8s}"
         )
         for task in self.tasks:
-            for element in task.elements:
+            for element in task.elements[:]:
                 for iter_idx, iteration in enumerate(element.iterations):
                     for act_idx, action_runs in iteration.actions.items():
                         for run_idx, EAR in enumerate(action_runs.runs):
+                            suc = EAR.success if EAR.success is not None else "-"
+                            if EAR.exit_code is not None:
+                                exc = f"{EAR.exit_code:^8d}"
+                            else:
+                                exc = f"{'-':^8}"
                             print(
                                 f"{task.insert_ID:^8d} {element.index:^8d} "
                                 f"{iter_idx:^8d} {act_idx:^8d} {run_idx:^8d} "
                                 f"{EAR.submission_status.name.lower():^8s}"
+                                f"{exc}"
+                                f"{suc:^8}"
+                                f"{EAR.skip:^8}"
                             )