hpcflow 0.1.9__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

hpcflow/sdk/persistence/base.py

@@ -0,0 +1,2817 @@
+"""
+Base persistence models.
+
+Store* classes represent the element-metadata in the store, in a store-agnostic way.
+"""
+
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from collections import defaultdict
+import contextlib
+import copy
+from dataclasses import dataclass, field
+import enum
+from logging import Logger
+from functools import wraps
+import os
+from pathlib import Path
+import shutil
+import socket
+import time
+from typing import Generic, TypeVar, cast, overload, TYPE_CHECKING
+
+import numpy as np
+
+from hpcflow.sdk.core.utils import (
+    flatten,
+    get_in_container,
+    get_relative_path,
+    remap,
+    reshape,
+    set_in_container,
+    normalise_timestamp,
+    parse_timestamp,
+    current_timestamp,
+)
+from hpcflow.sdk.core.errors import ParametersMetadataReadOnlyError
+from hpcflow.sdk.submission.submission import (
+    JOBSCRIPT_SUBMIT_TIME_KEYS,
+    SUBMISSION_SUBMIT_TIME_KEYS,
+)
+from hpcflow.sdk.utils.strings import shorten_list_str
+from hpcflow.sdk.log import TimeIt
+from hpcflow.sdk.typing import hydrate
+from hpcflow.sdk.persistence.pending import PendingChanges
+from hpcflow.sdk.persistence.types import (
+    AnySTask,
+    AnySElement,
+    AnySElementIter,
+    AnySEAR,
+    AnySParameter,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+    from contextlib import AbstractContextManager
+    from datetime import datetime
+    from typing import Any, ClassVar, Final, Literal
+    from typing_extensions import Self, TypeIs
+    from fsspec import AbstractFileSystem  # type: ignore
+    from numpy.typing import NDArray
+    from .pending import CommitResourceMap
+    from .store_resource import StoreResource
+    from .types import (
+        EncodedStoreParameter,
+        File,
+        FileDescriptor,
+        LoopDescriptor,
+        Metadata,
+        ParameterTypes,
+        PersistenceCache,
+        StoreCreationInfo,
+        TemplateMeta,
+        TypeLookup,
+        IterableParam,
+    )
+    from .zarr import ZarrAttrsDict
+    from ..app import BaseApp
+    from ..typing import DataIndex, PathLike, ParamSource
+    from ..core.json_like import JSONed, JSONDocument
+    from ..core.parameters import ParameterValue
+    from ..core.workflow import Workflow
+    from ..submission.types import VersionInfo, ResolvedJobscriptBlockDependencies
+
+T = TypeVar("T")
+#: Type of the serialized form.
+SerFormT = TypeVar("SerFormT")
+#: Type of the encoding and decoding context.
+ContextT = TypeVar("ContextT")
+
+PRIMITIVES = (
+    int,
+    float,
+    str,
+    type(None),
+)
+
+TEMPLATE_COMP_TYPES = (
+    "parameters",
+    "command_files",
+    "environments",
+    "task_schemas",
+)
+
+PARAM_DATA_NOT_SET: Final[int] = 0
+
+
+def update_param_source_dict(source: ParamSource, update: ParamSource) -> ParamSource:
+    """
+    Combine two dicts into a new dict that is ordered on its keys.
+    """
+    return cast("ParamSource", dict(sorted({**source, **update}.items())))
+
+
+def writes_parameter_data(func: Callable):
+    """Decorator function that should wrap `PersistentStore` methods that write
+    parameter-associated data.
+
+    Notes
+    -----
+    This decorator checks that the parameters-metadata cache is not in use, which should
+    not be used during writing of parameter-associated data.
+    """
+
+    @wraps(func)
+    def inner(self, *args, **kwargs):
+        if self._use_parameters_metadata_cache:
+            raise ParametersMetadataReadOnlyError(
+                "Cannot use the `parameters_metadata_cache` when writing parameter-"
+                "associated data!"
+            )
+        return func(self, *args, **kwargs)
+
+    return inner
+
+
+@dataclass
+class PersistentStoreFeatures:
+    """
+    Represents the features provided by a persistent store.
+
+    Parameters
+    ----------
+    create:
+        If True, a new workflow can be created using this store.
+    edit:
+        If True, the workflow can be modified.
+    jobscript_parallelism:
+        If True, the store supports workflows running multiple independent jobscripts
+        simultaneously.
+    EAR_parallelism:
+        If True, the store supports workflows running multiple EARs simultaneously.
+    schedulers:
+        If True, the store supports submitting workflows to a scheduler.
+    submission:
+        If True, the store supports submission. If False, the store can be considered to
+        be an archive, which would need transforming to another store type before
+        submission.
+    """
+
+    #: Whether a new workflow can be created using this store.
+    create: bool = False
+    #: Whether the workflow can be modified.
+    edit: bool = False
+    #: Whetherthe store supports workflows running multiple independent jobscripts
+    #: simultaneously.
+    jobscript_parallelism: bool = False
+    #: Whether the store supports workflows running multiple EARs simultaneously.
+    EAR_parallelism: bool = False
+    #: Whether the store supports submitting workflows to a scheduler.
+    schedulers: bool = False
+    #: Whether the store supports submission. If not, the store can be considered to
+    #: be an archive, which would need transforming to another store type before
+    #: submission.
+    submission: bool = False
+
+
+@dataclass
+class StoreTask(Generic[SerFormT]):
+    """
+    Represents a task in a persistent store.
+
+    Parameters
+    ----------
+    id_:
+        The ID of the task.
+    index:
+        The index of the task within its workflow.
+    is_pending:
+        Whether the task has changes not yet persisted.
+    element_IDs:
+        The IDs of elements in the task.
+    task_template:
+        Description of the template for the task.
+    """
+
+    # This would be in the docstring except it renders really wrongly!
+    # Type Parameters
+    # ---------------
+    # SerFormT
+    #     Type of the serialized form.
+
+    #: The ID of the task.
+    id_: int
+    #: The index of the task within its workflow.
+    index: int
+    #: Whether the task has changes not yet persisted.
+    is_pending: bool
+    #: The IDs of elements in the task.
+    element_IDs: list[int]
+    #: Description of the template for the task.
+    task_template: Mapping[str, Any] | None = None
+
+    @abstractmethod
+    def encode(self) -> tuple[int, SerFormT, dict[str, Any]]:
+        """Prepare store task data for the persistent store."""
+
+    @classmethod
+    @abstractmethod
+    def decode(cls, task_dat: SerFormT) -> Self:
+        """Initialise a `StoreTask` from store task data
+
+        Note: the `task_template` is only needed for encoding because it is retrieved as
+        part of the `WorkflowTemplate` so we don't need to load it when decoding.
+
+        """
+
+    @TimeIt.decorator
+    def append_element_IDs(self, pend_IDs: list[int]) -> Self:
+        """Return a copy, with additional element IDs."""
+        return self.__class__(
+            id_=self.id_,
+            index=self.index,
+            is_pending=self.is_pending,
+            element_IDs=[*self.element_IDs, *pend_IDs],
+            task_template=self.task_template,
+        )
+
+
+@dataclass
+class StoreElement(Generic[SerFormT, ContextT]):
+    """
+    Represents an element in a persistent store.
+
+    Parameters
+    ----------
+    id_:
+        The ID of the element.
+    is_pending:
+        Whether the element has changes not yet persisted.
+    index:
+        Index of the element within its parent task.
+    es_idx:
+        Index of the element set containing this element.
+    seq_idx:
+        Value sequence index map.
+    src_idx:
+        Data source index map.
+    task_ID:
+        ID of the task that contains this element.
+    iteration_IDs:
+        IDs of element-iterations that belong to this element.
+    """
+
+    # These would be in the docstring except they render really wrongly!
+    # Type Parameters
+    # ---------------
+    # SerFormT
+    #     Type of the serialized form.
+    # ContextT
+    #     Type of the encoding and decoding context.
+
+    #: The ID of the element.
+    id_: int
+    #: Whether the element has changes not yet persisted.
+    is_pending: bool
+    #: Index of the element within its parent task.
+    index: int
+    #: Index of the element set containing this element.
+    es_idx: int
+    #: Value sequence index map.
+    seq_idx: dict[str, int]
+    #: Data source index map.
+    src_idx: dict[str, int]
+    #: ID of the task that contains this element.
+    task_ID: int
+    #: IDs of element-iterations that belong to this element.
+    iteration_IDs: list[int]
+
+    @abstractmethod
+    def encode(self, context: ContextT) -> SerFormT:
+        """Prepare store element data for the persistent store."""
+
+    @classmethod
+    @abstractmethod
+    def decode(cls, elem_dat: SerFormT, context: ContextT) -> Self:
+        """Initialise a `StoreElement` from store element data"""
+
+    def to_dict(self, iters) -> dict[str, Any]:
+        """Prepare data for the user-facing `Element` object."""
+        return {
+            "id_": self.id_,
+            "is_pending": self.is_pending,
+            "index": self.index,
+            "es_idx": self.es_idx,
+            "seq_idx": self.seq_idx,
+            "src_idx": self.src_idx,
+            "iteration_IDs": self.iteration_IDs,
+            "task_ID": self.task_ID,
+            "iterations": iters,
+        }
+
+    @TimeIt.decorator
+    def append_iteration_IDs(self, pend_IDs: Iterable[int]) -> Self:
+        """Return a copy, with additional iteration IDs."""
+        iter_IDs = [*self.iteration_IDs, *pend_IDs]
+        return self.__class__(
+            id_=self.id_,
+            is_pending=self.is_pending,
+            index=self.index,
+            es_idx=self.es_idx,
+            seq_idx=self.seq_idx,
+            src_idx=self.src_idx,
+            task_ID=self.task_ID,
+            iteration_IDs=iter_IDs,
+        )
+
+
+@dataclass
+class StoreElementIter(Generic[SerFormT, ContextT]):
+    """
+    Represents an element iteration in a persistent store.
+
+    Parameters
+    ----------
+    id_:
+        The ID of this element iteration.
+    is_pending:
+        Whether the element iteration has changes not yet persisted.
+    element_ID:
+        Which element is an iteration for.
+    EARs_initialised:
+        Whether EARs have been initialised for this element iteration.
+    EAR_IDs:
+        Maps task schema action indices to EARs by ID.
+    data_idx:
+        Overall data index for the element-iteration, which maps parameter names to
+        parameter data indices.
+    schema_parameters:
+        List of parameters defined by the associated task schema.
+    loop_idx:
+        What loops are being handled here and where they're up to.
+    """
+
+    # These would be in the docstring except they render really wrongly!
+    # Type Parameters
+    # ---------------
+    # SerFormT
+    #     Type of the serialized form.
+    # ContextT
+    #     Type of the encoding and decoding context.
+
+    #: The ID of this element iteration.
+    id_: int
+    #: Whether the element iteration has changes not yet persisted.
+    is_pending: bool
+    #: Which element is an iteration for.
+    element_ID: int
+    #: Whether EARs have been initialised for this element iteration.
+    EARs_initialised: bool
+    #: Maps task schema action indices to EARs by ID.
+    EAR_IDs: dict[int, list[int]] | None
+    #: Overall data index for the element-iteration, which maps parameter names to
+    #: parameter data indices.
+    data_idx: DataIndex
+    #: List of parameters defined by the associated task schema.
+    schema_parameters: list[str]
+    #: What loops are being handled here and where they're up to.
+    loop_idx: Mapping[str, int] = field(default_factory=dict)
+
+    @abstractmethod
+    def encode(self, context: ContextT) -> SerFormT:
+        """Prepare store element iteration data for the persistent store."""
+
+    @classmethod
+    @abstractmethod
+    def decode(cls, iter_dat: SerFormT, context: ContextT) -> Self:
+        """Initialise a `StoreElementIter` from persistent store element iteration data"""
+
+    def to_dict(self, EARs: dict[int, dict[str, Any]] | None) -> dict[str, Any]:
+        """Prepare data for the user-facing `ElementIteration` object."""
+        return {
+            "id_": self.id_,
+            "is_pending": self.is_pending,
+            "element_ID": self.element_ID,
+            "EAR_IDs": self.EAR_IDs,
+            "data_idx": self.data_idx,
+            "schema_parameters": self.schema_parameters,
+            "EARs": EARs,
+            "EARs_initialised": self.EARs_initialised,
+            "loop_idx": dict(self.loop_idx),
+        }
+
+    @TimeIt.decorator
+    def append_EAR_IDs(self, pend_IDs: Mapping[int, Sequence[int]]) -> Self:
+        """Return a copy, with additional EAR IDs."""
+
+        EAR_IDs = copy.deepcopy(self.EAR_IDs) or {}
+        for act_idx, IDs_i in pend_IDs.items():
+            EAR_IDs.setdefault(act_idx, []).extend(IDs_i)
+
+        return self.__class__(
+            id_=self.id_,
+            is_pending=self.is_pending,
+            element_ID=self.element_ID,
+            EAR_IDs=EAR_IDs,
+            data_idx=self.data_idx,
+            schema_parameters=self.schema_parameters,
+            loop_idx=self.loop_idx,
+            EARs_initialised=self.EARs_initialised,
+        )
+
+    @TimeIt.decorator
+    def update_loop_idx(self, loop_idx: Mapping[str, int]) -> Self:
+        """Return a copy, with the loop index updated."""
+        loop_idx_new = dict(self.loop_idx)
+        loop_idx_new.update(loop_idx)
+        return self.__class__(
+            id_=self.id_,
+            is_pending=self.is_pending,
+            element_ID=self.element_ID,
+            EAR_IDs=self.EAR_IDs,
+            data_idx=self.data_idx,
+            schema_parameters=self.schema_parameters,
+            EARs_initialised=self.EARs_initialised,
+            loop_idx=loop_idx_new,
+        )
+
+    @TimeIt.decorator
+    def set_EARs_initialised(self) -> Self:
+        """Return a copy with `EARs_initialised` set to `True`."""
+        return self.__class__(
+            id_=self.id_,
+            is_pending=self.is_pending,
+            element_ID=self.element_ID,
+            EAR_IDs=self.EAR_IDs,
+            data_idx=self.data_idx,
+            schema_parameters=self.schema_parameters,
+            loop_idx=self.loop_idx,
+            EARs_initialised=True,
+        )
+
+    @TimeIt.decorator
+    def update_data_idx(self: AnySElementIter, data_idx: DataIndex) -> AnySElementIter:
+        """Return a copy with an updated `data_idx`.
+
+        The existing data index is updated, not overwritten.
+
+        """
+        new_data_idx = copy.deepcopy(self.data_idx)
+        new_data_idx.update(data_idx)
+        return self.__class__(
+            id_=self.id_,
+            is_pending=self.is_pending,
+            element_ID=self.element_ID,
+            EAR_IDs=self.EAR_IDs,
+            data_idx=new_data_idx,
+            schema_parameters=self.schema_parameters,
+            loop_idx=self.loop_idx,
+            EARs_initialised=self.EARs_initialised,
+        )
+
+
+@dataclass
+class StoreEAR(Generic[SerFormT, ContextT]):
+    """
+    Represents an element action run in a persistent store.
+
+    Parameters
+    ----------
+    id_:
+        The ID of this element action run.
+    is_pending:
+        Whether the element action run has changes not yet persisted.
+    elem_iter_ID:
+        What element iteration owns this EAR.
+    action_idx:
+        The task schema action associated with this EAR.
+    commands_idx:
+        The indices of the commands in the EAR.
+    data_idx:
+        Maps parameter names within this EAR to parameter data indices.
+    submission_idx:
+        Which submission contained this EAR, if known.
+    skip:
+        Whether to skip this EAR.
+    success:
+        Whether this EAR was successful, if known.
+    start_time:
+        When this EAR started, if known.
+    end_time:
+        When this EAR finished, if known.
+    snapshot_start:
+        Snapshot of files at EAR start, if recorded.
+    snapshot_end:
+        Snapshot of files at EAR end, if recorded.
+    exit_code:
+        The exit code of the underlying executable, if known.
+    metadata:
+        Metadata concerning e.g. the state of the EAR.
+    run_hostname:
+        Where this EAR was submitted to run, if known.
+    """
+
+    # These would be in the docstring except they render really wrongly!
+    # Type Parameters
+    # ---------------
+    # SerFormT
+    #     Type of the serialized form.
+    # ContextT
+    #     Type of the encoding and decoding context.
+
+    #: The ID of this element action run.
+    id_: int
+    #: Whether the element action run has changes not yet persisted.
+    is_pending: bool
+    #: What element iteration owns this EAR.
+    elem_iter_ID: int
+    #: The task schema action associated with this EAR.
+    action_idx: int
+    #: The indices of the commands in the EAR.
+    commands_idx: list[int]
+    #: Maps parameter names within this EAR to parameter data indices.
+    data_idx: DataIndex
+    #: Which submission contained this EAR, if known.
+    submission_idx: int | None = None
+    #: Run ID whose commands can be used for this run (may be this run's ID).
+    commands_file_ID: int | None = None
+    #: Whether to skip this EAR.
+    skip: int = 0
+    #: Whether this EAR was successful, if known.
+    success: bool | None = None
+    #: When this EAR started, if known.
+    start_time: datetime | None = None
+    #: When this EAR finished, if known.
+    end_time: datetime | None = None
+    #: Snapshot of files at EAR start, if recorded.
+    snapshot_start: dict[str, Any] | None = None
+    #: Snapshot of files at EAR end, if recorded.
+    snapshot_end: dict[str, Any] | None = None
+    #: The exit code of the underlying executable, if known.
+    exit_code: int | None = None
+    #: Metadata concerning e.g. the state of the EAR.
+    metadata: Metadata | None = None
+    #: Where this EAR was submitted to run, if known.
+    run_hostname: str | None = None
+    port_number: int | None = None
+
+    @staticmethod
+    def _encode_datetime(dt: datetime | None, ts_fmt: str) -> str | None:
+        return dt.strftime(ts_fmt) if dt else None
+
+    @staticmethod
+    def _decode_datetime(dt_str: str | None, ts_fmt: str) -> datetime | None:
+        return parse_timestamp(dt_str, ts_fmt) if dt_str else None
+
+    @abstractmethod
+    def encode(self, ts_fmt: str, context: ContextT) -> SerFormT:
+        """Prepare store EAR data for the persistent store."""
+
+    @classmethod
+    @abstractmethod
+    def decode(cls, EAR_dat: SerFormT, ts_fmt: str, context: ContextT) -> Self:
+        """Initialise a `StoreEAR` from persistent store EAR data"""
+
+    def to_dict(self) -> dict[str, Any]:
+        """Prepare data for the user-facing `ElementActionRun` object."""
+
+        def _process_datetime(dt: datetime | None) -> datetime | None:
+            """We store datetime objects implicitly in UTC, so we need to first make
+            that explicit, and then convert to the local time zone."""
+            return normalise_timestamp(dt) if dt else None
+
+        return {
+            "id_": self.id_,
+            "is_pending": self.is_pending,
+            "elem_iter_ID": self.elem_iter_ID,
+            "action_idx": self.action_idx,
+            "commands_idx": self.commands_idx,
+            "data_idx": self.data_idx,
+            "submission_idx": self.submission_idx,
+            "commands_file_ID": self.commands_file_ID,
+            "success": self.success,
+            "skip": self.skip,
+            "start_time": _process_datetime(self.start_time),
+            "end_time": _process_datetime(self.end_time),
+            "snapshot_start": self.snapshot_start,
+            "snapshot_end": self.snapshot_end,
+            "exit_code": self.exit_code,
+            "metadata": self.metadata,
+            "run_hostname": self.run_hostname,
+            "port_number": self.port_number,
+        }
+
+    @TimeIt.decorator
+    def update(
+        self,
+        submission_idx: int | None = None,
+        commands_file_ID: int | None = None,
+        skip: int | None = None,
+        success: bool | None = None,
+        start_time: datetime | None = None,
+        end_time: datetime | None = None,
+        snapshot_start: dict[str, Any] | None = None,
+        snapshot_end: dict[str, Any] | None = None,
+        exit_code: int | None = None,
+        run_hostname: str | None = None,
+        port_number: int | None = None,
+        data_idx: DataIndex | None = None,
+    ) -> Self:
+        """Return a shallow copy, with specified data updated."""
+
+        sub_idx = submission_idx if submission_idx is not None else self.submission_idx
+        skip = skip if skip is not None else self.skip
+        success = success if success is not None else self.success
+        start_time = start_time if start_time is not None else self.start_time
+        end_time = end_time if end_time is not None else self.end_time
+        snap_s = snapshot_start if snapshot_start is not None else self.snapshot_start
+        snap_e = snapshot_end if snapshot_end is not None else self.snapshot_end
+        exit_code = exit_code if exit_code is not None else self.exit_code
+        run_hn = run_hostname if run_hostname is not None else self.run_hostname
+        port_num = port_number if port_number is not None else self.port_number
+        cmd_file = (
+            commands_file_ID if commands_file_ID is not None else self.commands_file_ID
+        )
+        if data_idx is not None:
+            new_data_idx = copy.deepcopy(self.data_idx)
+            new_data_idx.update(data_idx)
+            data_idx = new_data_idx
+        else:
+            data_idx = self.data_idx
+
+        return self.__class__(
+            id_=self.id_,
+            is_pending=self.is_pending,
+            elem_iter_ID=self.elem_iter_ID,
+            action_idx=self.action_idx,
+            commands_idx=self.commands_idx,
+            data_idx=data_idx,
+            metadata=self.metadata,
+            submission_idx=sub_idx,
+            commands_file_ID=cmd_file,
+            skip=skip,
+            success=success,
+            start_time=start_time,
+            end_time=end_time,
+            snapshot_start=snap_s,
+            snapshot_end=snap_e,
+            exit_code=exit_code,
+            run_hostname=run_hn,
+            port_number=port_num,
+        )
+
+
+@dataclass
+@hydrate
+class StoreParameter:
+    """
+    Represents a parameter in a persistent store.
+
+    Parameters
+    ----------
+    id_:
+        The ID of this parameter.
+    is_pending:
+        Whether the parameter has changes not yet persisted.
+    is_set:
+        Whether the parameter is set.
+    data:
+        Description of the value of the parameter.
+    file:
+        Description of the file this parameter represents.
+    source:
+        Description of where this parameter originated.
+    """
+
+    #: The ID of this parameter.
+    id_: int
+    #: Whether the parameter has changes not yet persisted.
+    is_pending: bool
+    #: Whether the parameter is set.
+    is_set: bool
+    #: Description of the value of the parameter.
+    data: ParameterTypes
+    #: Description of the file this parameter represents.
+    file: File | None
+    #: Description of where this parameter originated.
+    source: ParamSource
+
+    _encoders: ClassVar[dict[type, Callable]] = {}
+    _decoders: ClassVar[dict[str, Callable]] = {}
+    _MAX_DEPTH: ClassVar[int] = 50
+
+    _all_encoders: ClassVar[dict[type, Callable]] = {}
+    _all_decoders: ClassVar[dict[str, Callable]] = {}
+
+    def encode(self, **kwargs) -> dict[str, Any] | int:
+        """Prepare store parameter data for the persistent store."""
+        if self.is_set:
+            if self.file:
+                return {"file": self.file}
+            else:
+                return cast("dict", self._encode(obj=self.data, **kwargs))
+        else:
+            return PARAM_DATA_NOT_SET
+
+    @staticmethod
+    def __is_ParameterValue(value) -> TypeIs[ParameterValue]:
+        # avoid circular import of `ParameterValue` until needed...
+        from ..core.parameters import ParameterValue as PV
+
+        return isinstance(value, PV)
+
+    def _encode(
+        self,
+        obj: ParameterTypes,
+        path: list[int] | None = None,
+        type_lookup: TypeLookup | None = None,
+        **kwargs,
+    ) -> EncodedStoreParameter:
+        """Recursive encoder."""
+
+        path = path or []
+        if type_lookup is None:
+            type_lookup = cast("TypeLookup", defaultdict(list))
+
+        if len(path) > self._MAX_DEPTH:
+            raise RuntimeError("I'm in too deep!")
+
+        if self.__is_ParameterValue(obj):
+            encoded = self._encode(
+                obj=obj.to_dict(),
+                path=path,
+                type_lookup=type_lookup,
+                **kwargs,
+            )
+            data, type_lookup = encoded["data"], encoded["type_lookup"]
+
+        elif isinstance(obj, (list, tuple, set)):
+            data = []
+            for idx, item in enumerate(obj):
+                encoded = self._encode(
+                    obj=item,
+                    path=[*path, idx],
+                    type_lookup=type_lookup,
+                    **kwargs,
+                )
+                item, type_lookup = encoded["data"], encoded["type_lookup"]
+                assert type_lookup is not None
+                data.append(item)
+
+            if isinstance(obj, tuple):
+                type_lookup["tuples"].append(path)
+
+            elif isinstance(obj, set):
+                type_lookup["sets"].append(path)
+
+        elif isinstance(obj, dict):
+            assert type_lookup is not None
+            data = {}
+            for dct_key, dct_val in obj.items():
+                encoded = self._encode(
+                    obj=dct_val,
+                    path=[*path, dct_key],
+                    type_lookup=type_lookup,
+                    **kwargs,
+                )
+                dct_val, type_lookup = encoded["data"], encoded["type_lookup"]
+                assert type_lookup is not None
+                data[dct_key] = dct_val
+
+        elif isinstance(obj, PRIMITIVES):
+            data = obj
+
+        elif type(obj) in self._all_encoders:
+            assert type_lookup is not None
+            data = self._all_encoders[type(obj)](
+                obj=obj,
+                path=path,
+                type_lookup=type_lookup,
+                root_encoder=self._encode,
+                **kwargs,
+            )
+
+        elif isinstance(obj, enum.Enum):
+            data = obj.value
+
+        else:
+            raise ValueError(
+                f"Parameter data with type {type(obj)} cannot be serialised into a "
+                f"{self.__class__.__name__}: {obj}."
+            )
+
+        return {"data": data, "type_lookup": type_lookup}
+
+    @classmethod
+    def decode(
+        cls,
+        id_: int,
+        data: dict[str, Any] | Literal[0] | None,
+        source: ParamSource,
+        *,
+        path: list[str] | None = None,
+        **kwargs,
+    ) -> Self:
+        """Initialise from persistent store parameter data."""
+        if data and "file" in data:
+            return cls(
+                id_=id_,
+                data=None,
+                file=cast("File", data["file"]),
+                is_set=True,
+                source=source,
+                is_pending=False,
+            )
+        elif not isinstance(data, dict):
+            # parameter is not set
+            return cls(
+                id_=id_,
+                data=None,
+                file=None,
+                is_set=False,
+                source=source,
+                is_pending=False,
+            )
+
+        data_ = cast("EncodedStoreParameter", data)
+        path = path or []
+
+        obj = get_in_container(data_["data"], path)
+
+        for type_, paths in data_["type_lookup"].items():
+            for type_path in paths:
+                if type_ == "tuples":
+                    try:
+                        rel_path = get_relative_path(type_path, path)
+                    except ValueError:
+                        continue
+                    if rel_path:
+                        set_in_container(
+                            obj, rel_path, tuple(get_in_container(obj, rel_path))
+                        )
+                    else:
+                        obj = tuple(obj)
+                elif type_ == "sets":
+                    try:
+                        rel_path = get_relative_path(type_path, path)
+                    except ValueError:
+                        continue
+                    if rel_path:
+                        set_in_container(
+                            obj, rel_path, set(get_in_container(obj, rel_path))
+                        )
+                    else:
+                        obj = set(obj)
+                elif type_ in cls._all_decoders:
+                    obj = cls._all_decoders[type_](
+                        obj=obj,
+                        type_lookup=data_["type_lookup"],
+                        path=path,
+                        **kwargs,
+                    )
+
+        return cls(
+            id_=id_,
+            data=obj,
+            file=None,
+            is_set=True,
+            source=source,
+            is_pending=False,
+        )
+
+    def set_data(self, value: Any) -> Self:
+        """Return a copy, with data set."""
+        if self.is_set:
+            raise RuntimeError(f"Parameter ID {self.id_!r} is already set!")
+        return self.__class__(
+            id_=self.id_,
+            is_set=True,
+            is_pending=self.is_pending,
+            data=value,
+            file=None,
+            source=self.source,
+        )
+
+    def set_file(self, value: File) -> Self:
+        """Return a copy, with file set."""
+        if self.is_set:
+            raise RuntimeError(f"Parameter ID {self.id_!r} is already set!")
+        return self.__class__(
+            id_=self.id_,
+            is_set=True,
+            is_pending=self.is_pending,
+            data=None,
+            file=value,
+            source=self.source,
+        )
+
+    def update_source(self, src: ParamSource) -> Self:
+        """Return a copy, with updated source."""
+        return self.__class__(
+            id_=self.id_,
+            is_set=self.is_set,
+            is_pending=self.is_pending,
+            data=self.data,
+            file=self.file,
+            source=update_param_source_dict(self.source, src),
+        )
+
+
+class PersistentStore(
+    ABC, Generic[AnySTask, AnySElement, AnySElementIter, AnySEAR, AnySParameter]
+):
+    """
+    An abstract class representing a persistent workflow store.
+
+    Parameters
+    ----------
+    app: App
+        The main hpcflow core.
+    workflow: ~hpcflow.app.Workflow
+        The workflow being persisted.
+    path: pathlib.Path
+        Where to hold the store.
+    fs: fsspec.AbstractFileSystem
+        Optionally, information about how to access the store.
+    """
+
+    # These would be in the docstring except they render really wrongly!
+    # Type Parameters
+    # ---------------
+    # AnySTask: StoreTask
+    #     The type of stored tasks.
+    # AnySElement: StoreElement
+    #     The type of stored elements.
+    # AnySElementIter: StoreElementIter
+    #     The type of stored element iterations.
+    # AnySEAR: StoreEAR
+    #     The type of stored EARs.
+    # AnySParameter: StoreParameter
+    #     The type of stored parameters.
+
+    _name: ClassVar[str]
+
+    @classmethod
+    @abstractmethod
+    def _store_task_cls(cls) -> type[AnySTask]: ...
+
+    @classmethod
+    @abstractmethod
+    def _store_elem_cls(cls) -> type[AnySElement]: ...
+
+    @classmethod
+    @abstractmethod
+    def _store_iter_cls(cls) -> type[AnySElementIter]: ...
+
+    @classmethod
+    @abstractmethod
+    def _store_EAR_cls(cls) -> type[AnySEAR]: ...
+
+    @classmethod
+    @abstractmethod
+    def _store_param_cls(cls) -> type[AnySParameter]: ...
+
+    _resources: dict[str, StoreResource]
+    _features: ClassVar[PersistentStoreFeatures]
+    _res_map: ClassVar[CommitResourceMap]
+
+    def __init__(
+        self,
+        app: BaseApp,
+        workflow: Workflow | None,
+        path: Path | str,
+        fs: AbstractFileSystem | None = None,
+    ):
+        self._app = app
+        self.__workflow = workflow
+        self.path = str(path)
+        self.fs = fs
+
+        self._pending: PendingChanges[
+            AnySTask, AnySElement, AnySElementIter, AnySEAR, AnySParameter
+        ] = PendingChanges(app=app, store=self, resource_map=self._res_map)
+
+        self._resources_in_use: set[tuple[str, str]] = set()
+        self._in_batch_mode = False
+
+        self._use_cache = False
+        self._reset_cache()
+
+        self._use_parameters_metadata_cache: bool = False  # subclass-specific cache
+
+    def _ensure_all_encoders(self):
+        """Ensure app-defined encoders are included in the StoreParameter's encoders
+        map."""
+        param_cls = self._store_param_cls()
+        if not param_cls._all_encoders:
+            param_cls._all_encoders = {
+                **param_cls._encoders,
+                **self.workflow._app.encoders().get(self._name, {}),
+            }
+
+    def _ensure_all_decoders(self):
+        """Ensure app-defined decoders are included in the StoreParameter's decoders
+        map."""
+        param_cls = self._store_param_cls()
+        if not param_cls._all_decoders:
+            param_cls._all_decoders = {
+                **param_cls._decoders,
+                **self.workflow._app.decoders().get(self._name, {}),
+            }
+
+    @abstractmethod
+    def cached_load(self) -> contextlib.AbstractContextManager[None]:
+        """
+        Perform a load with cache enabled while the ``with``-wrapped code runs.
+        """
+
+    @abstractmethod
+    def get_name(self) -> str:
+        """
+        Get the workflow name.
+        """
+
+    @abstractmethod
+    def get_creation_info(self) -> StoreCreationInfo:
+        """
+        Get the workflow creation data.
+        """
+
+    @abstractmethod
+    def get_ts_fmt(self) -> str:
+        """
+        Get the timestamp format.
+        """
+
+    @abstractmethod
+    def get_ts_name_fmt(self) -> str:
+        """
+        Get the timestamp format for names.
+        """
+
+    @abstractmethod
+    def remove_replaced_dir(self) -> None:
+        """
+        Remove a replaced directory.
+        """
+
+    @abstractmethod
+    def reinstate_replaced_dir(self) -> None:
+        """
+        Reinstate a replaced directory.
+        """
+
+    @abstractmethod
+    def zip(
+        self,
+        path: str = ".",
+        log: str | None = None,
+        overwrite=False,
+        include_execute=False,
+        include_rechunk_backups=False,
+    ) -> str:
+        """
+        Convert this store into archival form.
+        """
+
+    @abstractmethod
+    def unzip(self, path: str = ".", log: str | None = None) -> str:
+        """
+        Convert this store into expanded form.
+        """
+
+    @abstractmethod
+    def rechunk_parameter_base(
+        self,
+        chunk_size: int | None = None,
+        backup: bool = True,
+        status: bool = True,
+    ) -> Any: ...
+
+    @abstractmethod
+    def rechunk_runs(
+        self,
+        chunk_size: int | None = None,
+        backup: bool = True,
+        status: bool = True,
+    ) -> Any: ...
+
+    @abstractmethod
+    def get_dirs_array(self) -> NDArray:
+        """
+        Retrieve the run directories array.
+        """
+
+    @classmethod
+    @abstractmethod
+    def write_empty_workflow(
+        cls,
+        app: BaseApp,
+        *,
+        template_js: TemplateMeta,
+        template_components_js: dict[str, Any],
+        wk_path: str,
+        fs: AbstractFileSystem,
+        name: str,
+        replaced_wk: str | None,
+        creation_info: StoreCreationInfo,
+        ts_fmt: str,
+        ts_name_fmt: str,
+    ) -> None:
+        """
+        Write an empty workflow.
+        """
+
+    @property
+    def workflow(self) -> Workflow:
+        """
+        The workflow this relates to.
+        """
+        assert self.__workflow is not None
+        return self.__workflow
+
+    @property
+    def logger(self) -> Logger:
+        """
+        The logger to use.
+        """
+        return self._app.persistence_logger
+
+    @property
+    def ts_fmt(self) -> str:
+        """
+        The format for timestamps.
+        """
+        return self.workflow.ts_fmt
+
+    @property
+    def has_pending(self) -> bool:
+        """
+        Whether there are any pending changes.
+        """
+        return bool(self._pending)
+
+    @property
+    def is_submittable(self) -> bool:
+        """Does this store support workflow submission?"""
+        return self.fs.__class__.__name__ == "LocalFileSystem"
+
+    @property
+    def use_cache(self) -> bool:
+        """
+        Whether to use a cache.
+        """
+        return self._use_cache
+
+    @property
+    def task_cache(self) -> dict[int, AnySTask]:
+        """Cache for persistent tasks."""
+        return self._cache["tasks"]
+
+    @property
+    def element_cache(self) -> dict[int, AnySElement]:
+        """Cache for persistent elements."""
+        return self._cache["elements"]
+
+    @property
+    def element_iter_cache(self) -> dict[int, AnySElementIter]:
+        """Cache for persistent element iterations."""
+        return self._cache["element_iters"]
+
+    @property
+    def EAR_cache(self) -> dict[int, AnySEAR]:
+        """Cache for persistent EARs."""
+        return self._cache["EARs"]
+
+    @property
+    def num_tasks_cache(self) -> int | None:
+        """Cache for number of persistent tasks."""
+        return self._cache["num_tasks"]
+
+    @num_tasks_cache.setter
+    def num_tasks_cache(self, value: int | None):
+        self._cache["num_tasks"] = value
+
+    @property
+    def num_EARs_cache(self) -> int | None:
+        """Cache for total number of persistent EARs."""
+        return self._cache["num_EARs"]
+
+    @num_EARs_cache.setter
+    def num_EARs_cache(self, value: int | None):
+        self._cache["num_EARs"] = value
+
+    @property
+    def num_params_cache(self) -> int | None:
+        return self._cache["num_params"]
+
+    @num_params_cache.setter
+    def num_params_cache(self, value: int | None):
+        self._cache["num_params"] = value
+
+    @property
+    def param_sources_cache(self) -> dict[int, ParamSource]:
+        """Cache for persistent parameter sources."""
+        return self._cache["param_sources"]
+
+    @property
+    def parameter_cache(self) -> dict[int, AnySParameter]:
+        """Cache for persistent parameters."""
+        return self._cache["parameters"]
+
+    def _reset_cache(self) -> None:
+        self._cache: PersistenceCache[
+            AnySTask, AnySElement, AnySElementIter, AnySEAR, AnySParameter
+        ] = {
+            "tasks": {},
+            "elements": {},
+            "element_iters": {},
+            "EARs": {},
+            "param_sources": {},
+            "num_tasks": None,
+            "parameters": {},
+            "num_EARs": None,
+            "num_params": None,
+        }
+
+    @contextlib.contextmanager
+    def cache_ctx(self) -> Iterator[None]:
+        """Context manager for using the persistent element/iteration/run cache."""
+        if self._use_cache:
+            yield
+        else:
+            self._use_cache = True
+            self._reset_cache()
+            try:
+                yield
+            finally:
+                self._use_cache = False
+                self._reset_cache()
+
+    @contextlib.contextmanager
+    def parameters_metadata_cache(self):
+        """Context manager for using the parameters-metadata cache.
+
+        Notes
+        -----
+        This method can be overridden by a subclass to provide an implementation-specific
+        cache of metadata associated with parameters, or even parameter data itself.
+
+        Using this cache precludes writing/setting parameter data.
+
+        """
+        yield
+
+    @staticmethod
+    def prepare_test_store_from_spec(
+        task_spec: Sequence[
+            Mapping[str, Sequence[Mapping[str, Sequence[Mapping[str, Sequence]]]]]
+        ],
+    ) -> tuple[list[dict], list[dict], list[dict], list[dict]]:
+        """Generate a valid store from a specification in terms of nested
+        elements/iterations/EARs.
+
+        """
+        tasks: list[dict] = []
+        elements: list[dict] = []
+        elem_iters: list[dict] = []
+        EARs: list[dict] = []
+
+        for task_idx, task_i in enumerate(task_spec):
+            elems_i = task_i.get("elements", [])
+            elem_IDs = list(range(len(elements), len(elements) + len(elems_i)))
+
+            for elem_idx, elem_j in enumerate(elems_i):
+                iters_j = elem_j.get("iterations", [])
+                iter_IDs = list(range(len(elem_iters), len(elem_iters) + len(iters_j)))
+
+                for iter_k in iters_j:
+                    EARs_k = iter_k.get("EARs", [])
+                    EAR_IDs = list(range(len(EARs), len(EARs) + len(EARs_k)))
+                    EAR_IDs_dct = {0: EAR_IDs} if EAR_IDs else {}
+
+                    for _ in EARs_k:
+                        EARs.append(
+                            {
+                                "id_": len(EARs),
+                                "is_pending": False,
+                                "elem_iter_ID": len(elem_iters),
+                                "action_idx": 0,
+                                "data_idx": {},
+                                "metadata": {},
+                            }
+                        )
+
+                    elem_iters.append(
+                        {
+                            "id_": len(elem_iters),
+                            "is_pending": False,
+                            "element_ID": len(elements),
+                            "EAR_IDs": EAR_IDs_dct,
+                            "data_idx": {},
+                            "schema_parameters": [],
+                        }
+                    )
+                elements.append(
+                    {
+                        "id_": len(elements),
+                        "is_pending": False,
+                        "element_idx": elem_idx,
+                        "seq_idx": {},
+                        "src_idx": {},
+                        "task_ID": task_idx,
+                        "iteration_IDs": iter_IDs,
+                    }
+                )
+            tasks.append(
+                {
+                    "id_": len(tasks),
+                    "is_pending": False,
+                    "element_IDs": elem_IDs,
+                }
+            )
+        return (tasks, elements, elem_iters, EARs)
+
+    def remove_path(self, path: str | Path) -> None:
+        """Try very hard to delete a directory or file.
+
+        Dropbox (on Windows, at least) seems to try to re-sync files if the parent directory
+        is deleted soon after creation, which is the case on a failed workflow creation (e.g.
+        missing inputs), so in addition to catching PermissionErrors generated when
+        Dropbox has a lock on files, we repeatedly try deleting the directory tree.
+
+        """
+
+        fs = self.fs
+        assert fs is not None
+
+        @self._app.perm_error_retry()
+        def _remove_path(_path: str) -> None:
+            self.logger.debug(f"_remove_path: path={_path}")
+            while fs.exists(_path):
+                fs.rm(_path, recursive=True)
+                time.sleep(0.5)
+
+        return _remove_path(str(path))
+
+    def rename_path(self, replaced: str, original: str | Path) -> None:
+        """Revert the replaced workflow path to its original name.
+
+        This happens when new workflow creation fails and there is an existing workflow
+        with the same name; the original workflow which was renamed, must be reverted."""
+
+        fs = self.fs
+        assert fs is not None
+
+        @self._app.perm_error_retry()
+        def _rename_path(_replaced: str, _original: str) -> None:
+            self.logger.debug(f"_rename_path: {_replaced!r} --> {_original!r}.")
+            try:
+                fs.rename(
+                    _replaced, _original, recursive=True
+                )  # TODO: why need recursive?
+            except TypeError:
+                # `SFTPFileSystem.rename` has no  `recursive` argument:
+                fs.rename(_replaced, _original)
+
+        return _rename_path(str(replaced), str(original))
+
+    @abstractmethod
+    def _get_num_persistent_tasks(self) -> int: ...
+
+    def _get_num_total_tasks(self) -> int:
+        """Get the total number of persistent and pending tasks."""
+        return self._get_num_persistent_tasks() + len(self._pending.add_tasks)
+
+    @abstractmethod
+    def _get_num_persistent_loops(self) -> int: ...
+
+    def _get_num_total_loops(self) -> int:
+        """Get the total number of persistent and pending loops."""
+        return self._get_num_persistent_loops() + len(self._pending.add_loops)
+
+    @abstractmethod
+    def _get_num_persistent_submissions(self) -> int: ...
+
+    def _get_num_total_submissions(self) -> int:
+        """Get the total number of persistent and pending submissions."""
+        return self._get_num_persistent_submissions() + len(self._pending.add_submissions)
+
+    @abstractmethod
+    def _get_num_persistent_elements(self) -> int: ...
+
+    def _get_num_total_elements(self) -> int:
+        """Get the total number of persistent and pending elements."""
+        return self._get_num_persistent_elements() + len(self._pending.add_elements)
+
+    @abstractmethod
+    def _get_num_persistent_elem_iters(self) -> int: ...
+
+    def _get_num_total_elem_iters(self) -> int:
+        """Get the total number of persistent and pending element iterations."""
+        return self._get_num_persistent_elem_iters() + len(self._pending.add_elem_iters)
+
+    @abstractmethod
+    def _get_num_persistent_EARs(self) -> int: ...
+
+    @TimeIt.decorator
+    def _get_num_total_EARs(self) -> int:
+        """Get the total number of persistent and pending EARs."""
+        return self._get_num_persistent_EARs() + len(self._pending.add_EARs)
+
+    def _get_task_total_num_elements(self, task_ID: int) -> int:
+        """Get the total number of persistent and pending elements of a given task."""
+        return len(self.get_task(task_ID).element_IDs)
+
+    @abstractmethod
+    def _get_num_persistent_parameters(self) -> int: ...
+
+    def _get_num_total_parameters(self) -> int:
+        """Get the total number of persistent and pending parameters."""
+        return self._get_num_persistent_parameters() + len(self._pending.add_parameters)
+
+    def _get_num_total_input_files(self) -> int:
+        """Get the total number of persistent and pending user-supplied input files."""
+        return self._get_num_persistent_input_files() + sum(
+            fd["is_input"] for fd in self._pending.add_files
+        )
+
+    @abstractmethod
+    def _get_num_persistent_added_tasks(self) -> int: ...
+
+    def _get_num_total_added_tasks(self) -> int:
+        """Get the total number of tasks ever added to the workflow."""
+        return self._get_num_persistent_added_tasks() + len(self._pending.add_tasks)
+
+    def _get_num_persistent_input_files(self) -> int:
+        return sum(1 for _ in self.workflow.input_files_path.glob("*"))
+
+    def save(self) -> None:
+        """Commit pending changes to disk, if not in batch-update mode."""
+        if not self.workflow._in_batch_mode:
+            self._pending.commit_all()
+
+    def add_template_components(
+        self, temp_comps: Mapping[str, dict], save: bool = True
+    ) -> None:
+        """
+        Add template components to the workflow.
+        """
+        all_tc = self.get_template_components()
+        for name, dat in temp_comps.items():
+            if name in all_tc:
+                for hash_i, dat_i in dat.items():
+                    if hash_i not in all_tc[name]:
+                        self._pending.add_template_components[name][hash_i] = dat_i
+            else:
+                self._pending.add_template_components[name] = dat
+
+        if save:
+            self.save()
+
+    def add_task(self, idx: int, task_template: Mapping, save: bool = True):
+        """Add a new task to the workflow."""
+        self.logger.debug("Adding store task.")
+        new_ID = self._get_num_total_added_tasks()
+        self._pending.add_tasks[new_ID] = self._store_task_cls()(
+            id_=new_ID,
+            index=idx,
+            task_template=task_template,
+            is_pending=True,
+            element_IDs=[],
+        )
+        if save:
+            self.save()
+        return new_ID
+
+    def add_loop(
+        self,
+        loop_template: Mapping[str, Any],
+        iterable_parameters: Mapping[str, IterableParam],
+        output_parameters: Mapping[str, int],
+        parents: Sequence[str],
+        num_added_iterations: Mapping[tuple[int, ...], int],
+        iter_IDs: Iterable[int],
+        save: bool = True,
+    ):
+        """Add a new loop to the workflow."""
+        self.logger.debug("Adding store loop.")
+        new_idx = self._get_num_total_loops()
+        added_iters: list[list[list[int] | int]] = [
+            [list(k), v] for k, v in num_added_iterations.items()
+        ]
+        self._pending.add_loops[new_idx] = {
+            "loop_template": dict(loop_template),
+            "iterable_parameters": cast("dict", iterable_parameters),
+            "output_parameters": cast("dict", output_parameters),
+            "parents": list(parents),
+            "num_added_iterations": added_iters,
+        }
+
+        for i in iter_IDs:
+            self._pending.update_loop_indices[i][loop_template["name"]] = 0
+
+        if save:
+            self.save()
+
+    @TimeIt.decorator
+    def add_submission(
+        self, sub_idx: int, sub_js: Mapping[str, JSONed], save: bool = True
+    ):
+        """Add a new submission."""
+        self.logger.debug("Adding store submission.")
+        self._pending.add_submissions[sub_idx] = sub_js
+        if save:
+            self.save()
+
+    def add_element_set(self, task_id: int, es_js: Mapping, save: bool = True):
+        """
+        Add an element set to a task.
+        """
+        self._pending.add_element_sets[task_id].append(es_js)
+        if save:
+            self.save()
+
+    def add_element(
+        self,
+        task_ID: int,
+        es_idx: int,
+        seq_idx: dict[str, int],
+        src_idx: dict[str, int],
+        save: bool = True,
+    ) -> int:
+        """Add a new element to a task."""
+        self.logger.debug("Adding store element.")
+        new_ID = self._get_num_total_elements()
+        new_elem_idx = self._get_task_total_num_elements(task_ID)
+        self._pending.add_elements[new_ID] = self._store_elem_cls()(
+            id_=new_ID,
+            is_pending=True,
+            index=new_elem_idx,
+            es_idx=es_idx,
+            seq_idx=seq_idx,
+            src_idx=src_idx,
+            task_ID=task_ID,
+            iteration_IDs=[],
+        )
+        self._pending.add_elem_IDs[task_ID].append(new_ID)
+        if save:
+            self.save()
+        return new_ID
+
+    def add_element_iteration(
+        self,
+        element_ID: int,
+        data_idx: DataIndex,
+        schema_parameters: list[str],
+        loop_idx: Mapping[str, int] | None = None,
+        save: bool = True,
+    ) -> int:
+        """Add a new iteration to an element."""
+        self.logger.debug("Adding store element-iteration.")
+        new_ID = self._get_num_total_elem_iters()
+        self._pending.add_elem_iters[new_ID] = self._store_iter_cls()(
+            id_=new_ID,
+            element_ID=element_ID,
+            is_pending=True,
+            EARs_initialised=False,
+            EAR_IDs=None,
+            data_idx=data_idx,
+            schema_parameters=schema_parameters,
+            loop_idx=loop_idx or {},
+        )
+        self._pending.add_elem_iter_IDs[element_ID].append(new_ID)
+        if save:
+            self.save()
+        return new_ID
+
+    @TimeIt.decorator
+    def add_EAR(
+        self,
+        elem_iter_ID: int,
+        action_idx: int,
+        commands_idx: list[int],
+        data_idx: DataIndex,
+        metadata: Metadata | None = None,
+        save: bool = True,
+    ) -> int:
+        """Add a new EAR to an element iteration."""
+        self.logger.debug("Adding store EAR.")
+        new_ID = self._get_num_total_EARs()
+        self._pending.add_EARs[new_ID] = self._store_EAR_cls()(
+            id_=new_ID,
+            is_pending=True,
+            elem_iter_ID=elem_iter_ID,
+            action_idx=action_idx,
+            commands_idx=commands_idx,
+            data_idx=data_idx,
+            metadata=metadata or {},
+        )
+        self._pending.add_elem_iter_EAR_IDs[elem_iter_ID][action_idx].append(new_ID)
+        if save:
+            self.save()
+        return new_ID
+
+    @TimeIt.decorator
+    def set_run_dirs(
+        self, run_dir_indices: np.ndarray, run_idx: np.ndarray, save: bool = True
+    ):
+        self.logger.debug(f"Setting {run_idx.size} run directory indices.")
+        self._pending.set_run_dirs.append((run_dir_indices, run_idx))
+        if save:
+            self.save()
+
+    def update_at_submit_metadata(
+        self, sub_idx: int, submission_parts: dict[str, list[int]], save: bool = True
+    ):
+        """
+        Update metadata that is set at submit-time.
+        """
+        if submission_parts:
+            self._pending.update_at_submit_metadata[sub_idx][
+                "submission_parts"
+            ] = submission_parts
+        if save:
+            self.save()
+
+    @TimeIt.decorator
+    def set_run_submission_data(
+        self, EAR_ID: int, cmds_ID: int | None, sub_idx: int, save: bool = True
+    ) -> None:
+        """
+        Set the run submission data, like the submission index for an element action run.
+        """
+        self._pending.set_EAR_submission_data[EAR_ID] = (sub_idx, cmds_ID)
+        if save:
+            self.save()
+
+    def set_EAR_start(
+        self,
+        EAR_ID: int,
+        run_dir: Path | None,
+        port_number: int | None,
+        save: bool = True,
+    ) -> datetime:
+        """
+        Mark an element action run as started.
+        """
+        dt = current_timestamp()
+        ss_js = self._app.RunDirAppFiles.take_snapshot() if run_dir else None
+        run_hostname = socket.gethostname()
+        self._pending.set_EAR_starts[EAR_ID] = (dt, ss_js, run_hostname, port_number)
+        if save:
+            self.save()
+        return dt
+
+    def set_multi_run_starts(
+        self,
+        run_ids: list[int],
+        run_dirs: list[Path | None],
+        port_number: int,
+        save: bool = True,
+    ) -> datetime:
+        dt = current_timestamp()
+        run_hostname = socket.gethostname()
+        run_start_data: dict[int, tuple] = {}
+        for id_i, dir_i in zip(run_ids, run_dirs):
+            ss_js_i = self._app.RunDirAppFiles.take_snapshot(dir_i) if dir_i else None
+            run_start_data[id_i] = (dt, ss_js_i, run_hostname, port_number)
+
+        self._pending.set_EAR_starts.update(run_start_data)
+        if save:
+            self.save()
+        return dt
+
+    def set_EAR_end(
+        self,
+        EAR_ID: int,
+        exit_code: int,
+        success: bool,
+        snapshot: bool,
+        save: bool = True,
+    ) -> datetime:
+        """
+        Mark an element action run as finished.
+        """
+        # TODO: save output files
+        dt = current_timestamp()
+        ss_js = self._app.RunDirAppFiles.take_snapshot() if snapshot else None
+        self._pending.set_EAR_ends[EAR_ID] = (dt, ss_js, exit_code, success)
+        if save:
+            self.save()
+        return dt
+
+    def set_multi_run_ends(
+        self,
+        run_ids: list[int],
+        run_dirs: list[Path | None],
+        exit_codes: list[int],
+        successes: list[bool],
+        save: bool = True,
+    ) -> datetime:
+        self.logger.info("PersistentStore.set_multi_run_ends.")
+        dt = current_timestamp()
+        run_end_data: dict[int, tuple] = {}
+        for id_i, dir_i, ex_i, sc_i in zip(run_ids, run_dirs, exit_codes, successes):
+            ss_js_i = self._app.RunDirAppFiles.take_snapshot(dir_i) if dir_i else None
+            run_end_data[id_i] = (dt, ss_js_i, ex_i, sc_i)
+
+        self._pending.set_EAR_ends.update(run_end_data)
+        if save:
+            self.save()
+        self.logger.info("PersistentStore.set_multi_run_ends finished.")
+        return dt
+
+    def set_EAR_skip(self, skip_reasons: dict[int, int], save: bool = True) -> None:
+        """
+        Mark element action runs as skipped for the specified reasons.
+        """
+        self._pending.set_EAR_skips.update(skip_reasons)
+        if save:
+            self.save()
+
+    def set_EARs_initialised(self, iter_ID: int, save: bool = True) -> None:
+        """
+        Mark an element action run as initialised.
+        """
+        self._pending.set_EARs_initialised.append(iter_ID)
+        if save:
+            self.save()
+
+    def set_jobscript_metadata(
+        self,
+        sub_idx: int,
+        js_idx: int,
+        version_info: VersionInfo | None = None,
+        submit_time: str | None = None,
+        submit_hostname: str | None = None,
+        submit_machine: str | None = None,
+        shell_idx: int | None = None,
+        submit_cmdline: list[str] | None = None,
+        os_name: str | None = None,
+        shell_name: str | None = None,
+        scheduler_name: str | None = None,
+        scheduler_job_ID: str | None = None,
+        process_ID: int | None = None,
+        save: bool = True,
+    ):
+        """
+        Set the metadata for a job script.
+        """
+        entry = self._pending.set_js_metadata[sub_idx][js_idx]
+        if version_info:
+            entry["version_info"] = version_info
+        if submit_time:
+            entry["submit_time"] = submit_time
+        if submit_hostname:
+            entry["submit_hostname"] = submit_hostname
+        if submit_machine:
+            entry["submit_machine"] = submit_machine
+        if shell_idx is not None:
+            entry["shell_idx"] = shell_idx
+        if submit_cmdline:
+            entry["submit_cmdline"] = submit_cmdline
+        if os_name:
+            entry["os_name"] = os_name
+        if shell_name:
+            entry["shell_name"] = shell_name
+        if scheduler_name:
+            entry["scheduler_name"] = scheduler_name
+        if scheduler_job_ID or process_ID:
+            entry["scheduler_job_ID"] = scheduler_job_ID
+        if process_ID or scheduler_job_ID:
+            entry["process_ID"] = process_ID
+        if save:
+            self.save()
+
+    @writes_parameter_data
+    def _add_parameter(
+        self,
+        is_set: bool,
+        source: ParamSource,
+        data: (
+            ParameterValue | list | tuple | set | dict | int | float | str | None | Any
+        ) = None,
+        file: File | None = None,
+        save: bool = True,
+    ) -> int:
+        self.logger.debug(f"Adding store parameter{f' (unset)' if not is_set else ''}.")
+        new_idx = self._get_num_total_parameters()
+        self._pending.add_parameters[new_idx] = self._store_param_cls()(
+            id_=new_idx,
+            is_pending=True,
+            is_set=is_set,
+            data=PARAM_DATA_NOT_SET if not is_set else data,
+            file=file,
+            source=source,
+        )
+        if save:
+            self.save()
+        return new_idx
+
+    def _prepare_set_file(
+        self,
+        store_contents: bool,
+        is_input: bool,
+        path: Path | str,
+        contents: str | None = None,
+        filename: str | None = None,
+        clean_up: bool = False,
+    ) -> File:
+        if filename is None:
+            filename = Path(path).name
+
+        if store_contents:
+            if is_input:
+                new_idx = self._get_num_total_input_files()
+                dst_dir = Path(self.workflow.input_files_path, str(new_idx))
+                dst_path = dst_dir / filename
+            else:
+                # assume path is inside the EAR execution directory; transform that to the
+                # equivalent artifacts directory:
+                exec_sub_path = Path(path).relative_to(self.path)
+                dst_path = Path(
+                    self.workflow.task_artifacts_path, *exec_sub_path.parts[1:]
+                )
+            if dst_path.is_file():
+                dst_path = dst_path.with_suffix(dst_path.suffix + "_2")  # TODO: better!
+        else:
+            dst_path = Path(path)
+
+        file_param_dat: File = {
+            "store_contents": store_contents,
+            "path": str(dst_path.relative_to(self.path)),
+        }
+        self._pending.add_files.append(
+            {
+                "store_contents": store_contents,
+                "is_input": is_input,
+                "dst_path": str(dst_path),
+                "path": str(path),
+                "contents": contents or "",
+                "clean_up": clean_up,
+            }
+        )
+
+        return file_param_dat
+
+    def set_file(
+        self,
+        store_contents: bool,
+        is_input: bool,
+        param_id: int | None,
+        path: Path | str,
+        contents: str | None = None,
+        filename: str | None = None,
+        clean_up: bool = False,
+        save: bool = True,
+    ):
+        """
+        Set details of a file, including whether it is associated with a parameter.
+        """
+        self.logger.debug("Setting new file")
+        file_param_dat = self._prepare_set_file(
+            store_contents=store_contents,
+            is_input=is_input,
+            path=path,
+            contents=contents,
+            filename=filename,
+            clean_up=clean_up,
+        )
+        if param_id is not None:
+            self.set_parameter_value(
+                param_id, value=file_param_dat, is_file=True, save=save
+            )
+        if save:
+            self.save()
+
+    def add_file(
+        self,
+        store_contents: bool,
+        is_input: bool,
+        source: ParamSource,
+        path: Path | str,
+        contents: str | None = None,
+        filename: str | None = None,
+        save: bool = True,
+    ):
+        """
+        Add a file that will be associated with a parameter.
+        """
+        self.logger.debug("Adding new file")
+        file_param_dat = self._prepare_set_file(
+            store_contents=store_contents,
+            is_input=is_input,
+            path=path,
+            contents=contents,
+            filename=filename,
+        )
+        p_id = self._add_parameter(
+            file=file_param_dat,
+            is_set=True,
+            source=source,
+            save=save,
+        )
+        if save:
+            self.save()
+        return p_id
+
+    def _append_files(self, files: list[FileDescriptor]):
+        """Add new files to the files or artifacts directories."""
+        for dat in files:
+            if dat["store_contents"]:
+                dst_path = Path(dat["dst_path"])
+                dst_path.parent.mkdir(parents=True, exist_ok=True)
+                if dat["path"] is not None:
+                    # copy from source path to destination:
+                    shutil.copy(dat["path"], dst_path)
+                    if dat["clean_up"]:
+                        self.logger.info(f"deleting file {dat['path']}")
+                        os.remove(dat["path"])
+                else:
+                    # write out text file:
+                    with dst_path.open("wt") as fp:
+                        fp.write(dat["contents"])
+
+    @writes_parameter_data
+    def add_set_parameter(
+        self,
+        data: ParameterValue | list | tuple | set | dict | int | float | str | Any,
+        source: ParamSource,
+        save: bool = True,
+    ) -> int:
+        """
+        Add a parameter that is set to a value.
+        """
+        return self._add_parameter(data=data, is_set=True, source=source, save=save)
+
+    @writes_parameter_data
+    def add_unset_parameter(self, source: ParamSource, save: bool = True) -> int:
+        """
+        Add a parameter that is not set to any value.
+        """
+        return self._add_parameter(data=None, is_set=False, source=source, save=save)
+
+    @abstractmethod
+    def _set_parameter_values(self, set_parameters: dict[int, tuple[Any, bool]]): ...
+
+    @writes_parameter_data
+    def set_parameter_value(
+        self, param_id: int, value: Any, is_file: bool = False, save: bool = True
+    ):
+        """
+        Set the value of a parameter.
+        """
+        self.logger.debug(
+            f"Setting store parameter ID {param_id} value with type: {type(value)!r})."
+        )
+        self._pending.set_parameters[param_id] = (value, is_file)
+        if save:
+            self.save()
+
+    @writes_parameter_data
+    def set_parameter_values(self, values: dict[int, Any], save: bool = True):
+        """Set multiple non-file parameter values by parameter IDs."""
+        param_ids = values.keys()
+        self.logger.debug(f"Setting multiple store parameter IDs {param_ids!r}.")
+        self._pending.set_parameters.update({k: (v, False) for k, v in values.items()})
+        if save:
+            self.save()
+
+    @TimeIt.decorator
+    @writes_parameter_data
+    def update_param_source(
+        self, param_sources: Mapping[int, ParamSource], save: bool = True
+    ) -> None:
+        """
+        Set the source of a parameter.
+        """
+        self.logger.debug(f"Updating parameter sources with {param_sources!r}.")
+        self._pending.update_param_sources.update(param_sources)
+        if save:
+            self.save()
+
+    def update_loop_num_iters(
+        self,
+        index: int,
+        num_added_iters: Mapping[tuple[int, ...], int],
+        save: bool = True,
+    ) -> None:
+        """
+        Add iterations to a loop.
+        """
+        self.logger.debug(
+            f"Updating loop {index!r} num added iterations to {num_added_iters!r}."
+        )
+        self._pending.update_loop_num_iters[index] = [
+            [list(k), v] for k, v in num_added_iters.items()
+        ]
+        if save:
+            self.save()
+
+    def update_loop_parents(
+        self,
+        index: int,
+        num_added_iters: Mapping[tuple[int, ...], int],
+        parents: Sequence[str],
+        save: bool = True,
+    ) -> None:
+        """
+        Set the parents of a loop.
+        """
+        self.logger.debug(
+            f"Updating loop {index!r} parents to {parents!r}, and num added iterations "
+            f"to {num_added_iters}."
+        )
+        self._pending.update_loop_num_iters[index] = [
+            [list(k), v] for k, v in num_added_iters.items()
+        ]
+        self._pending.update_loop_parents[index] = list(parents)
+        if save:
+            self.save()
+
+    def update_iter_data_indices(self, data_indices: dict[int, DataIndex]):
+        """Update data indices of one or more iterations."""
+        for k, v in data_indices.items():
+            self._pending.update_iter_data_idx[k].update(v)
+
+    def update_run_data_indices(self, data_indices: dict[int, DataIndex]):
+        """Update data indices of one or more runs."""
+        for k, v in data_indices.items():
+            self._pending.update_run_data_idx[k].update(v)
+
+    def get_template_components(self) -> dict[str, Any]:
+        """Get all template components, including pending."""
+        tc = copy.deepcopy(self._get_persistent_template_components())
+        for typ in TEMPLATE_COMP_TYPES:
+            for hash_i, dat_i in self._pending.add_template_components.get(
+                typ, {}
+            ).items():
+                tc.setdefault(typ, {})[hash_i] = dat_i
+
+        return tc
+
+    @abstractmethod
+    def _get_persistent_template_components(self) -> dict[str, Any]: ...
+
+    def get_template(self) -> dict[str, JSONed]:
+        """
+        Get the workflow template.
+        """
+        return self._get_persistent_template()
+
+    @abstractmethod
+    def _get_persistent_template(self) -> dict[str, JSONed]: ...
+
+    def _get_task_id_to_idx_map(self) -> dict[int, int]:
+        return {task.id_: task.index for task in self.get_tasks()}
+
+    @TimeIt.decorator
+    def get_task(self, task_idx: int) -> AnySTask:
+        """
+        Get a task.
+        """
+        return self.get_tasks()[task_idx]
+
+    def __process_retrieved_tasks(self, tasks: Iterable[AnySTask]) -> list[AnySTask]:
+        """Add pending data to retrieved tasks."""
+        tasks_new: list[AnySTask] = []
+        for task in tasks:
+            # consider pending element IDs:
+            if pend_elems := self._pending.add_elem_IDs.get(task.id_):
+                task = task.append_element_IDs(pend_elems)
+            tasks_new.append(task)
+        return tasks_new
+
+    def __process_retrieved_loops(
+        self, loops: Iterable[tuple[int, LoopDescriptor]]
+    ) -> dict[int, LoopDescriptor]:
+        """Add pending data to retrieved loops."""
+        loops_new: dict[int, LoopDescriptor] = {}
+        for id_, loop_i in loops:
+            if "num_added_iterations" not in loop_i:
+                loop_i["num_added_iterations"] = 1
+            # consider pending changes to num added iterations:
+            if pend_num_iters := self._pending.update_loop_num_iters.get(id_):
+                loop_i["num_added_iterations"] = pend_num_iters
+            # consider pending change to parents:
+            if pend_parents := self._pending.update_loop_parents.get(id_):
+                loop_i["parents"] = pend_parents
+
+            loops_new[id_] = loop_i
+        return loops_new
+
+    @staticmethod
+    def __split_pending(
+        ids: Iterable[int], all_pending: Mapping[int, Any]
+    ) -> tuple[tuple[int, ...], set[int], set[int]]:
+        id_all = tuple(ids)
+        id_set = set(id_all)
+        id_pers = id_set.difference(all_pending)
+        id_pend = id_set.intersection(all_pending)
+        return id_all, id_pers, id_pend
+
+    @abstractmethod
+    def _get_persistent_tasks(self, id_lst: Iterable[int]) -> dict[int, AnySTask]: ...
+
+    def get_tasks_by_IDs(self, ids: Iterable[int]) -> Sequence[AnySTask]:
+        """
+        Get tasks with the given IDs.
+        """
+        # separate pending and persistent IDs:
+
+        ids, id_pers, id_pend = self.__split_pending(ids, self._pending.add_tasks)
+        tasks = self._get_persistent_tasks(id_pers) if id_pers else {}
+        tasks.update((id_, self._pending.add_tasks[id_]) for id_ in id_pend)
+
+        # order as requested:
+        return self.__process_retrieved_tasks(tasks[id_] for id_ in ids)
+
+    @TimeIt.decorator
+    def get_tasks(self) -> list[AnySTask]:
+        """Retrieve all tasks, including pending."""
+        tasks = self._get_persistent_tasks(range(self._get_num_persistent_tasks()))
+        tasks.update(self._pending.add_tasks)
+
+        # order by index:
+        return self.__process_retrieved_tasks(
+            sorted(tasks.values(), key=lambda x: x.index)
+        )
+
+    @abstractmethod
+    def _get_persistent_loops(
+        self, id_lst: Iterable[int] | None = None
+    ) -> dict[int, LoopDescriptor]: ...
+
+    def get_loops_by_IDs(self, ids: Iterable[int]) -> dict[int, LoopDescriptor]:
+        """Retrieve loops by index (ID), including pending."""
+
+        # separate pending and persistent IDs:
+        ids, id_pers, id_pend = self.__split_pending(ids, self._pending.add_loops)
+
+        loops = self._get_persistent_loops(id_pers) if id_pers else {}
+        loops.update((id_, self._pending.add_loops[id_]) for id_ in id_pend)
+
+        # order as requested:
+        return self.__process_retrieved_loops((id_, loops[id_]) for id_ in ids)
+
+    def get_loops(self) -> dict[int, LoopDescriptor]:
+        """Retrieve all loops, including pending."""
+
+        loops = self._get_persistent_loops()
+        loops.update(self._pending.add_loops)
+
+        # order by index/ID:
+        return self.__process_retrieved_loops(sorted(loops.items()))
+
+    @abstractmethod
+    def _get_persistent_submissions(
+        self, id_lst: Iterable[int] | None = None
+    ) -> dict[int, Mapping[str, JSONed]]: ...
+
+    @TimeIt.decorator
+    def get_submissions(self) -> dict[int, Mapping[str, JSONed]]:
+        """Retrieve all submissions, including pending."""
+
+        subs = self._get_persistent_submissions()
+        subs.update(self._pending.add_submissions)
+
+        # order by index/ID
+        return dict(sorted(subs.items()))
+
+    @TimeIt.decorator
+    def get_submission_at_submit_metadata(
+        self, sub_idx: int, metadata_attr: dict[str, Any] | None
+    ) -> dict[str, Any]:
+        """Retrieve the values of submission attributes that are stored at submit-time.
+
+        Notes
+        -----
+        This method may need to be overridden if these attributes are stored separately
+        from the remainder of the submission attributes.
+
+        """
+        return metadata_attr or {i: None for i in SUBMISSION_SUBMIT_TIME_KEYS}
+
+    @TimeIt.decorator
+    def get_jobscript_at_submit_metadata(
+        self,
+        sub_idx: int,
+        js_idx: int,
+        metadata_attr: dict[str, Any] | None,
+    ) -> dict[str, Any]:
+        """For the specified jobscript, retrieve the values of jobscript-submit-time
+        attributes.
+
+        Notes
+        -----
+        This method may need to be overridden if these jobscript-submit-time attributes
+        are stored separately from the remainder of the jobscript attributes.
+
+        """
+        return metadata_attr or {i: None for i in JOBSCRIPT_SUBMIT_TIME_KEYS}
+
+    @TimeIt.decorator
+    def get_jobscript_block_run_ID_array(
+        self, sub_idx: int, js_idx: int, blk_idx: int, run_ID_arr: NDArray | None
+    ) -> NDArray:
+        """For the specified jobscript-block, retrieve the run ID array.
+
+        Notes
+        -----
+        This method may need to be overridden if these attributes are stored separately
+        from the remainder of the submission attributes.
+
+        """
+        assert run_ID_arr is not None
+        return np.asarray(run_ID_arr)
+
+    @TimeIt.decorator
+    def get_jobscript_block_task_elements_map(
+        self,
+        sub_idx: int,
+        js_idx: int,
+        blk_idx: int,
+        task_elems_map: dict[int, list[int]] | None,
+    ) -> dict[int, list[int]]:
+        """For the specified jobscript-block, retrieve the task-elements mapping.
+
+        Notes
+        -----
+        This method may need to be overridden if these attributes are stored separately
+        from the remainder of the submission attributes.
+
+        """
+        assert task_elems_map is not None
+        return task_elems_map
+
+    @TimeIt.decorator
+    def get_jobscript_block_task_actions_array(
+        self,
+        sub_idx: int,
+        js_idx: int,
+        blk_idx: int,
+        task_actions_arr: NDArray | list[tuple[int, int, int]] | None,
+    ) -> NDArray:
+        """For the specified jobscript-block, retrieve the task-actions array.
+
+        Notes
+        -----
+        This method may need to be overridden if these attributes are stored separately
+        from the remainder of the submission attributes.
+
+        """
+        assert task_actions_arr is not None
+        return np.asarray(task_actions_arr)
+
+    @TimeIt.decorator
+    def get_jobscript_block_dependencies(
+        self,
+        sub_idx: int,
+        js_idx: int,
+        blk_idx: int,
+        js_dependencies: dict[tuple[int, int], ResolvedJobscriptBlockDependencies] | None,
+    ) -> dict[tuple[int, int], ResolvedJobscriptBlockDependencies]:
+        """For the specified jobscript-block, retrieve the dependencies.
+
+        Notes
+        -----
+        This method may need to be overridden if these attributes are stored separately
+        from the remainder of the submission attributes.
+
+        """
+        assert js_dependencies is not None
+        return js_dependencies
+
+    @TimeIt.decorator
+    def get_submissions_by_ID(
+        self, ids: Iterable[int]
+    ) -> dict[int, Mapping[str, JSONed]]:
+        """
+        Get submissions with the given IDs.
+        """
+        # separate pending and persistent IDs:
+        _, id_pers, id_pend = self.__split_pending(ids, self._pending.add_submissions)
+        subs = self._get_persistent_submissions(id_pers) if id_pers else {}
+        subs.update((id_, self._pending.add_submissions[id_]) for id_ in id_pend)
+
+        # order by index/ID
+        return dict(sorted(subs.items()))
+
+    @abstractmethod
+    def _get_persistent_elements(
+        self, id_lst: Iterable[int]
+    ) -> dict[int, AnySElement]: ...
+
+    @TimeIt.decorator
+    def get_elements(self, ids: Iterable[int]) -> Sequence[AnySElement]:
+        """
+        Get elements with the given IDs.
+        """
+        # separate pending and persistent IDs:
+        ids, id_pers, id_pend = self.__split_pending(ids, self._pending.add_elements)
+        self.logger.debug(
+            f"PersistentStore.get_elements: {len(ids)} elements: "
+            f"{shorten_list_str(ids)}."
+        )
+        elems = self._get_persistent_elements(id_pers) if id_pers else {}
+        elems.update((id_, self._pending.add_elements[id_]) for id_ in id_pend)
+
+        elems_new: list[AnySElement] = []
+        # order as requested:
+        for elem_i in (elems[id_] for id_ in ids):
+            # consider pending iteration IDs:
+            # TODO: does this consider pending iterations from new loop iterations?
+            if pend_iters := self._pending.add_elem_iter_IDs.get(elem_i.id_):
+                elem_i = elem_i.append_iteration_IDs(pend_iters)
+            elems_new.append(elem_i)
+
+        return elems_new
+
+    @abstractmethod
+    def _get_persistent_element_iters(
+        self, id_lst: Iterable[int]
+    ) -> dict[int, AnySElementIter]: ...
+
+    @TimeIt.decorator
+    def get_element_iterations(self, ids: Iterable[int]) -> Sequence[AnySElementIter]:
+        """
+        Get element iterations with the given IDs.
+        """
+        # separate pending and persistent IDs:
+        ids, id_pers, id_pend = self.__split_pending(ids, self._pending.add_elem_iters)
+        self.logger.debug(
+            f"PersistentStore.get_element_iterations: {len(ids)} iterations: "
+            f"{shorten_list_str(ids)}."
+        )
+        iters = self._get_persistent_element_iters(id_pers) if id_pers else {}
+        iters.update((id_, self._pending.add_elem_iters[id_]) for id_ in id_pend)
+
+        iters_new: list[AnySElementIter] = []
+        # order as requested:
+        for iter_i in (iters[id_] for id_ in ids):
+            # consider pending EAR IDs:
+            if pend_EARs := self._pending.add_elem_iter_EAR_IDs.get(iter_i.id_):
+                iter_i = iter_i.append_EAR_IDs(pend_EARs)
+
+            # consider pending loop idx
+            if pend_loop_idx := self._pending.update_loop_indices.get(iter_i.id_):
+                iter_i = iter_i.update_loop_idx(pend_loop_idx)
+
+            # consider pending `EARs_initialised`:
+            if iter_i.id_ in self._pending.set_EARs_initialised:
+                iter_i = iter_i.set_EARs_initialised()
+
+            iters_new.append(iter_i)
+
+        return iters_new
+
+    @abstractmethod
+    def _get_persistent_EARs(self, id_lst: Iterable[int]) -> dict[int, AnySEAR]: ...
+
+    @TimeIt.decorator
+    def get_EARs(self, ids: Iterable[int]) -> Sequence[AnySEAR]:
+        """
+        Get element action runs with the given IDs.
+        """
+        # separate pending and persistent IDs:
+        ids, id_pers, id_pend = self.__split_pending(ids, self._pending.add_EARs)
+        self.logger.debug(
+            f"PersistentStore.get_EARs: {len(ids)} EARs: {shorten_list_str(ids)}."
+        )
+        EARs = self._get_persistent_EARs(id_pers) if id_pers else {}
+        EARs.update((id_, self._pending.add_EARs[id_]) for id_ in id_pend)
+
+        EARs_new: list[AnySEAR] = []
+        # order as requested:
+        for EAR_i in (EARs[id_] for id_ in ids):
+            # consider updates:
+            updates: dict[str, Any] = {}
+            if EAR_i.id_ in self._pending.set_EAR_skips:
+                updates["skip"] = True
+            (
+                updates["submission_idx"],
+                updates["commands_file_ID"],
+            ) = self._pending.set_EAR_submission_data.get(EAR_i.id_, (None, None))
+            (
+                updates["start_time"],
+                updates["snapshot_start"],
+                updates["run_hostname"],
+                updates["port_number"],
+            ) = self._pending.set_EAR_starts.get(EAR_i.id_, (None, None, None, None))
+            (
+                updates["end_time"],
+                updates["snapshot_end"],
+                updates["exit_code"],
+                updates["success"],
+            ) = self._pending.set_EAR_ends.get(EAR_i.id_, (None, None, None, None))
+            if any(i is not None for i in updates.values()):
+                EAR_i = EAR_i.update(**updates)
+
+            EARs_new.append(EAR_i)
+
+        return EARs_new
+
+    @TimeIt.decorator
+    def __get_cached_persistent_items(
+        self, id_lst: Iterable[int], cache: dict[int, T]
+    ) -> tuple[dict[int, T], list[int]]:
+        """How to get things out of the cache. Caller says which cache."""
+        if self.use_cache:
+            id_cached = set(id_lst)
+            id_non_cached = sorted(id_cached.difference(cache))
+            id_cached.intersection_update(cache)
+            items = {id_: cache[id_] for id_ in sorted(id_cached)}
+        else:
+            items = {}
+            id_non_cached = list(id_lst)
+        return items, id_non_cached
+
+    def _get_cached_persistent_EARs(
+        self, id_lst: Iterable[int]
+    ) -> tuple[dict[int, AnySEAR], list[int]]:
+        return self.__get_cached_persistent_items(id_lst, self.EAR_cache)
+
+    def _get_cached_persistent_element_iters(
+        self, id_lst: Iterable[int]
+    ) -> tuple[dict[int, AnySElementIter], list[int]]:
+        return self.__get_cached_persistent_items(id_lst, self.element_iter_cache)
+
+    def _get_cached_persistent_elements(
+        self, id_lst: Iterable[int]
+    ) -> tuple[dict[int, AnySElement], list[int]]:
+        return self.__get_cached_persistent_items(id_lst, self.element_cache)
+
+    def _get_cached_persistent_tasks(
+        self, id_lst: Iterable[int]
+    ) -> tuple[dict[int, AnySTask], list[int]]:
+        return self.__get_cached_persistent_items(id_lst, self.task_cache)
+
+    def _get_cached_persistent_param_sources(
+        self, id_lst: Iterable[int]
+    ) -> tuple[dict[int, ParamSource], list[int]]:
+        return self.__get_cached_persistent_items(id_lst, self.param_sources_cache)
+
+    def _get_cached_persistent_parameters(
+        self, id_lst: Iterable[int]
+    ) -> tuple[dict[int, AnySParameter], list[int]]:
+        return self.__get_cached_persistent_items(id_lst, self.parameter_cache)
+
+    def get_EAR_skipped(self, EAR_ID: int) -> int:
+        """
+        Whether the element action run with the given ID was skipped.
+        """
+        self.logger.debug(f"PersistentStore.get_EAR_skipped: EAR_ID={EAR_ID!r}")
+        return self.get_EARs((EAR_ID,))[0].skip
+
+    @TimeIt.decorator
+    def get_parameters(self, ids: Iterable[int], **kwargs) -> list[AnySParameter]:
+        """
+        Get parameters with the given IDs.
+
+        Parameters
+        ----------
+        ids:
+            The IDs of the parameters to get.
+
+        Keyword Arguments
+        -----------------
+        dataset_copy: bool
+            For Zarr stores only. If True, copy arrays as NumPy arrays.
+        """
+        # separate pending and persistent IDs:
+        ids, id_pers, id_pend = self.__split_pending(ids, self._pending.add_parameters)
+        params = (
+            dict(self._get_persistent_parameters(id_pers, **kwargs)) if id_pers else {}
+        )
+        params.update((id_, self._pending.add_parameters[id_]) for id_ in id_pend)
+
+        # order as requested:
+        return [params[id_] for id_ in ids]
+
+    @abstractmethod
+    def _get_persistent_parameters(
+        self, id_lst: Iterable[int], **kwargs
+    ) -> Mapping[int, AnySParameter]: ...
+
+    @TimeIt.decorator
+    def get_parameter_set_statuses(self, ids: Iterable[int]) -> list[bool]:
+        """
+        Get whether the parameters with the given IDs are set.
+        """
+        # separate pending and persistent IDs:
+        ids, id_pers, id_pend = self.__split_pending(ids, self._pending.add_parameters)
+        set_status = self._get_persistent_parameter_set_status(id_pers) if id_pers else {}
+        set_status.update(
+            (id_, self._pending.add_parameters[id_].is_set) for id_ in id_pend
+        )
+
+        # order as requested:
+        return [set_status[id_] for id_ in ids]
+
+    @abstractmethod
+    def _get_persistent_parameter_set_status(
+        self, id_lst: Iterable[int]
+    ) -> dict[int, bool]: ...
+
+    @TimeIt.decorator
+    def get_parameter_sources(self, ids: Iterable[int]) -> list[ParamSource]:
+        """
+        Get the sources of the parameters with the given IDs.
+        """
+        # separate pending and persistent IDs:
+        ids, id_pers, id_pend = self.__split_pending(ids, self._pending.add_parameters)
+        src = self._get_persistent_param_sources(id_pers) if id_pers else {}
+        src.update((id_, self._pending.add_parameters[id_].source) for id_ in id_pend)
+
+        # order as requested, and consider pending source updates:
+        return [
+            self.__merge_param_source(
+                src[id_i], self._pending.update_param_sources.get(id_i)
+            )
+            for id_i in ids
+        ]
+
+    @staticmethod
+    def __merge_param_source(
+        src_i: ParamSource, pend_src: ParamSource | None
+    ) -> ParamSource:
+        """
+        Helper to merge a second dict in if it is provided.
+        """
+        return {**src_i, **pend_src} if pend_src else src_i
+
+    @abstractmethod
+    def _get_persistent_param_sources(
+        self, id_lst: Iterable[int]
+    ) -> dict[int, ParamSource]: ...
+
+    @TimeIt.decorator
+    def get_task_elements(
+        self,
+        task_id: int,
+        idx_lst: Iterable[int] | None = None,
+    ) -> Iterator[Mapping[str, Any]]:
+        """
+        Get element data by an indices within a given task.
+
+        Element iterations and EARs belonging to the elements are included.
+        """
+
+        all_elem_IDs = self.get_task(task_id).element_IDs
+        store_elements = self.get_elements(
+            all_elem_IDs if idx_lst is None else (all_elem_IDs[idx] for idx in idx_lst)
+        )
+        iter_IDs_flat, iter_IDs_lens = flatten(
+            [el.iteration_IDs for el in store_elements]
+        )
+        store_iters = self.get_element_iterations(iter_IDs_flat)
+
+        # retrieve EARs:
+        EARs_dcts = remap(
+            [list((elit.EAR_IDs or {}).values()) for elit in store_iters],
+            lambda ears: [ear.to_dict() for ear in self.get_EARs(ears)],
+        )
+
+        # add EARs to iterations:
+        iters: list[dict[str, Any]] = []
+        for idx, i in enumerate(store_iters):
+            EARs: dict[int, dict[str, Any]] | None = None
+            if i.EAR_IDs is not None:
+                EARs = dict(zip(i.EAR_IDs, cast("Any", EARs_dcts[idx])))
+            iters.append(i.to_dict(EARs))
+
+        # reshape iterations:
+        iters_rs = reshape(iters, iter_IDs_lens)
+
+        # add iterations to elements:
+        for idx, element in enumerate(store_elements):
+            yield element.to_dict(iters_rs[idx])
+
+    @abstractmethod
+    def _get_persistent_parameter_IDs(self) -> Iterable[int]: ...
+
+    def check_parameters_exist(self, ids: Sequence[int]) -> Iterator[bool]:
+        """
+        For each parameter ID, return True if it exists, else False.
+        """
+        id_miss = set()
+        if id_not_pend := set(ids).difference(self._pending.add_parameters):
+            id_miss = id_not_pend.difference(self._get_persistent_parameter_IDs())
+        return (id_ not in id_miss for id_ in ids)
+
+    @abstractmethod
+    def _append_tasks(self, tasks: Iterable[AnySTask]) -> None: ...
+
+    @abstractmethod
+    def _append_loops(self, loops: dict[int, LoopDescriptor]) -> None: ...
+
+    @abstractmethod
+    def _append_submissions(self, subs: dict[int, Mapping[str, JSONed]]) -> None: ...
+
+    @abstractmethod
+    def _update_at_submit_metadata(
+        self, at_submit_metadata: dict[int, dict[str, Any]]
+    ) -> None: ...
+
+    @abstractmethod
+    def _append_elements(self, elems: Sequence[AnySElement]) -> None: ...
+
+    @abstractmethod
+    def _append_element_sets(self, task_id: int, es_js: Sequence[Mapping]) -> None: ...
+
+    @abstractmethod
+    def _append_elem_iter_IDs(self, elem_ID: int, iter_IDs: Iterable[int]) -> None: ...
+
+    @abstractmethod
+    def _append_elem_iters(self, iters: Sequence[AnySElementIter]) -> None: ...
+
+    @abstractmethod
+    def _append_elem_iter_EAR_IDs(
+        self, iter_ID: int, act_idx: int, EAR_IDs: Sequence[int]
+    ) -> None: ...
+
+    @abstractmethod
+    def _append_EARs(self, EARs: Sequence[AnySEAR]) -> None: ...
+
+    @abstractmethod
+    def _update_elem_iter_EARs_initialised(self, iter_ID: int) -> None: ...
+
+    @abstractmethod
+    def _update_EAR_submission_data(
+        self, sub_data: Mapping[int, tuple[int, int | None]]
+    ): ...
+
+    @abstractmethod
+    def _update_EAR_start(
+        self,
+        run_starts: dict[int, tuple[datetime, dict[str, Any] | None, str, int | None]],
+    ) -> None: ...
+
+    @abstractmethod
+    def _update_EAR_end(
+        self, run_ends: dict[int, tuple[datetime, dict[str, Any] | None, int, bool]]
+    ) -> None: ...
+
+    @abstractmethod
+    def _update_EAR_skip(self, skips: dict[int, int]) -> None: ...
+
+    @abstractmethod
+    def _update_js_metadata(
+        self, js_meta: dict[int, dict[int, dict[str, Any]]]
+    ) -> None: ...
+
+    @abstractmethod
+    def _append_parameters(self, params: Sequence[AnySParameter]) -> None: ...
+
+    @abstractmethod
+    def _update_template_components(self, tc: dict[str, Any]) -> None: ...
+
+    @abstractmethod
+    def _update_parameter_sources(self, sources: Mapping[int, ParamSource]) -> None: ...
+
+    @abstractmethod
+    def _update_loop_index(self, loop_indices: dict[int, dict[str, int]]) -> None: ...
+
+    @abstractmethod
+    def _update_loop_num_iters(
+        self, index: int, num_iters: list[list[list[int] | int]]
+    ) -> None: ...
+
+    @abstractmethod
+    def _update_loop_parents(self, index: int, parents: list[str]) -> None: ...
+
+    @overload
+    def using_resource(
+        self, res_label: Literal["metadata"], action: str
+    ) -> AbstractContextManager[Metadata]: ...
+
+    @overload
+    def using_resource(
+        self, res_label: Literal["submissions"], action: str
+    ) -> AbstractContextManager[list[dict[str, JSONed]]]: ...
+
+    @overload
+    def using_resource(
+        self, res_label: Literal["parameters"], action: str
+    ) -> AbstractContextManager[dict[str, dict[str, Any]]]: ...
+
+    @overload
+    def using_resource(
+        self, res_label: Literal["runs"], action: str
+    ) -> AbstractContextManager[dict[str, Any]]: ...
+
+    @overload
+    def using_resource(
+        self, res_label: Literal["attrs"], action: str
+    ) -> AbstractContextManager[ZarrAttrsDict]: ...
+
+    @contextlib.contextmanager
+    def using_resource(
+        self,
+        res_label: Literal["metadata", "submissions", "parameters", "attrs", "runs"],
+        action: str,
+    ) -> Iterator[Any]:
+        """Context manager for managing `StoreResource` objects associated with the store."""
+
+        try:
+            res = self._resources[res_label]
+        except KeyError:
+            raise RuntimeError(
+                f"{self.__class__.__name__!r} has no resource named {res_label!r}."
+            ) from None
+
+        key = (res_label, action)
+        if key in self._resources_in_use:
+            # retrieve existing data for this action:
+            yield res.data[action]
+
+        else:
+            try:
+                # "open" the resource, which assigns data for this action, which we yield:
+                res.open(action)
+                self._resources_in_use.add(key)
+                yield res.data[action]
+
+            except Exception as exc:
+                self._resources_in_use.remove(key)
+                raise exc
+
+            else:
+                # "close" the resource, clearing cached data for this action:
+                res.close(action)
+                self._resources_in_use.remove(key)
+
+    def copy(self, path: PathLike = None) -> Path:
+        """Copy the workflow store.
+
+        This does not work on remote filesystems.
+
+        """
+        assert self.fs is not None
+        if path is None:
+            _path = Path(self.path)
+            path = _path.parent / Path(_path.stem + "_copy" + _path.suffix)
+
+        if self.fs.exists(str(path)):
+            raise ValueError(f"Path already exists: {path}.")
+        else:
+            path = str(path)
+
+        self.fs.copy(self.path, path)
+
+        return Path(self.workflow._store.path).replace(path)
+
+    def delete(self) -> None:
+        """Delete the persistent workflow."""
+        confirm = input(
+            f"Permanently delete the workflow at path {self.path!r}; [y]es or [n]o?"
+        )
+        if confirm.strip().lower() == "y":
+            self.delete_no_confirm()
+
+    def delete_no_confirm(self) -> None:
+        """Permanently delete the workflow data with no confirmation."""
+
+        fs = self.fs
+        assert fs is not None
+
+        @self._app.perm_error_retry()
+        def _delete_no_confirm() -> None:
+            self.logger.debug(f"_delete_no_confirm: {self.path!r}.")
+            fs.rm(self.path, recursive=True)
+
+        return _delete_no_confirm()
+
+    def get_text_file(self, path: str | Path) -> str:
+        """Retrieve the contents of a text file stored within the workflow.
+
+        Parameters
+        ----------
+        path
+            The path to a text file stored within the workflow. This can either be an
+            absolute path or a path that is relative to the workflow root.
+        """
+        path = Path(path)
+        if not path.is_absolute():
+            path = Path(self.path).joinpath(path)
+        if not path.is_file():
+            raise FileNotFoundError(f"File at location {path!r} does not exist.")
+        return path.read_text()
+
+    @abstractmethod
+    def _append_task_element_IDs(self, task_ID: int, elem_IDs: list[int]):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _set_run_dirs(self, run_dir_arr: np.ndarray, run_idx: np.ndarray) -> None: ...
+
+    @abstractmethod
+    def _update_iter_data_indices(
+        self, iter_data_indices: dict[int, DataIndex]
+    ) -> None: ...
+
+    @abstractmethod
+    def _update_run_data_indices(
+        self, run_data_indices: dict[int, DataIndex]
+    ) -> None: ...