hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

hpcflow/sdk/persistence/pending.py

@@ -5,17 +5,53 @@ Class to hold the state that is waiting to be committed to disk.
 from __future__ import annotations
 
 from collections import defaultdict
+from collections.abc import Callable
 import contextlib
 from dataclasses import dataclass, field, fields
-from datetime import datetime
+from functools import wraps
+import copy
 
-from typing import Any, Dict, List, Optional, Tuple
-
-from hpcflow.sdk.log import TimeIt
+from typing import Any, Generic, TYPE_CHECKING, TypeVar
+from typing_extensions import ParamSpec
 
+import numpy as np
 
-class PendingChanges:
-    """Class to store pending changes and merge them into a persistent store.
+from hpcflow.sdk.log import TimeIt
+from hpcflow.sdk.submission.submission import SUBMISSION_SUBMIT_TIME_KEYS
+from hpcflow.sdk.typing import DataIndex
+from hpcflow.sdk.utils.strings import shorten_list_str
+
+
+from hpcflow.sdk.persistence.types import (
+    AnySTask,
+    AnySElement,
+    AnySElementIter,
+    AnySEAR,
+    AnySParameter,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+    from datetime import datetime
+    from logging import Logger
+    from .base import PersistentStore, FileDescriptor, LoopDescriptor
+    from ..app import BaseApp
+    from ..typing import ParamSource
+    from ..core.json_like import JSONed
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+_commit_method_data_map: dict[str, list[str]] = defaultdict(
+    list
+)  # note: this is updated at module-import time.
+
+
+class PendingChanges(
+    Generic[AnySTask, AnySElement, AnySElementIter, AnySEAR, AnySParameter]
+):
+    """
+    Class to store pending changes and merge them into a persistent store.
 
     Parameters
     ----------
@@ -27,70 +63,104 @@ class PendingChanges:
         Map of resources, used when processing commits.
     """
 
-    def __init__(self, app, store, resource_map):
-        self.app = app
+    # These would be in the docstring except they render really wrongly!
+    # Type Parameters
+    # ---------------
+    # AnySTask
+    #     The type of stored tasks.
+    # AnySElement
+    #     The type of stored elements.
+    # AnySElementIter
+    #     The type of stored element iterations.
+    # AnySEAR
+    #     The type of stored EARs.
+    # AnySParameter
+    #     The type of stored parameters.
+
+    def __init__(
+        self,
+        app: BaseApp,
+        store: PersistentStore[
+            AnySTask, AnySElement, AnySElementIter, AnySEAR, AnySParameter
+        ],
+        resource_map: CommitResourceMap,
+    ):
+        self._app = app
         self.store = store
         self.resource_map = resource_map
 
         #: Keys are new task IDs.
-        self.add_tasks: Dict[int, AnySTask] = None
+        self.add_tasks: dict[int, AnySTask] = {}
         #: Keys are loop IDs, values are loop descriptors.
-        self.add_loops: Dict[int, Dict] = None
+        self.add_loops: dict[int, LoopDescriptor] = {}
         #: Keys are submission IDs, values are submission descriptors.
-        self.add_submissions: Dict[int, Dict] = None
+        self.add_submissions: dict[int, Mapping[str, JSONed]] = {}
         #: Keys are element IDs.
-        self.add_elements: Dict[int, AnySElement] = None
+        self.add_elements: dict[int, AnySElement] = {}
         #: Keys are element iteration IDs.
-        self.add_elem_iters: Dict[int, AnySElementIter] = None
+        self.add_elem_iters: dict[int, AnySElementIter] = {}
         #: Keys are element action run IDs.
-        self.add_EARs: Dict[int, AnySEAR] = None
+        self.add_EARs: dict[int, AnySEAR] = {}
         #: Keys are parameter indices and values are tuples whose first element is data
         #: to add and whose second element is the source dict for the new data.
-        self.add_parameters: Dict[int, AnySParameter] = None
+        self.add_parameters: dict[int, AnySParameter] = {}
         #: Workflow-related files (inputs, outputs) added to the persistent store.
-        self.add_files: List[Dict] = None
+        self.add_files: list[FileDescriptor] = []
         #: Template components to add.
-        self.add_template_components: Dict[str, Dict[str, Dict]] = None
+        self.add_template_components: dict[str, dict[str, dict]] = {}
         #: Keys are element set IDs, values are descriptors.
-        self.add_element_sets: Dict[int, Dict] = None
+        self.add_element_sets: dict[int, list[Mapping]] = {}
 
         #: Keys are task IDs, and values are element IDs to add to that task.
-        self.add_elem_IDs: Dict[int, List] = None
+        self.add_elem_IDs: dict[int, list[int]] = {}
         #: Keys are element IDs, and values are iteration IDs to add to that element.
-        self.add_elem_iter_IDs: Dict[int, List] = None
+        self.add_elem_iter_IDs: dict[int, list[int]] = {}
         #: Keys are element iteration IDs, then EAR action index, and values are EAR IDs.
         #: This is a list of EAR IDs to add to a given element iteration action.
-        self.add_elem_iter_EAR_IDs: Dict[int, Dict[int, List]] = None
-        #: Submission parts to add.
-        self.add_submission_parts: Dict[int, Dict[str, List[int]]] = None
+        self.add_elem_iter_EAR_IDs: dict[int, dict[int, list[int]]] = {}
+        #: Submission metadata added at submit-time, including submission parts.
+        self.update_at_submit_metadata: dict[int, dict[str, Any]] = {}
 
         #: IDs of EARs to mark as initialised.
-        self.set_EARs_initialised: List[int] = None
-        #: Submission IDs to attach to EARs.
-        self.set_EAR_submission_indices: Dict[int, int] = None
+        self.set_EARs_initialised: list[int] = []
+        #: Submission IDs and commands file IDs to attach to EARs.
+        self.set_EAR_submission_data: dict[int, tuple[int, int | None]] = {}
         #: IDs of EARs to mark as skipped.
-        self.set_EAR_skips: List[int] = None
-        #: Keys are EAR IDs and values are tuples of start time, and start dir snapshot.
-        self.set_EAR_starts: Dict[int, Tuple[datetime, Dict], str] = None
+        self.set_EAR_skips: dict[int, int] = {}
+        #: Keys are EAR IDs and values are tuples of start time, start dir snapshot, run
+        #: hostname, and port number.
+        self.set_EAR_starts: dict[
+            int, tuple[datetime, dict[str, Any] | None, str, int | None]
+        ] = {}
         #: Keys are EAR IDs and values are tuples of end time, end dir snapshot, exit
         #: code, and success boolean.
-        self.set_EAR_ends: Dict[int, Tuple[datetime, Dict, int, bool]] = None
+        self.set_EAR_ends: dict[
+            int, tuple[datetime, dict[str, Any] | None, int, bool]
+        ] = {}
+        #: Each list item is a tuple of two arrays, the first of which is a run directory
+        #: indices array, and the second of which is an integer array indicating with
+        #: which run ID each run directory is associated.
+        self.set_run_dirs: list[tuple[np.ndarray, np.ndarray]] = []
 
         #: Keys are IDs of jobscripts.
-        self.set_js_metadata: Dict[int, Dict[int, Any]] = None
+        self.set_js_metadata: dict[int, dict[int, dict[str, Any]]] = {}
 
-        #: Keys are IDs of parameters to add or modify.
-        self.set_parameters: Dict[int, AnySParameter] = None
+        #: Keys are IDs of parameters to add or modify, and values are tuples of the
+        #: parameter value, and whether the parameter is a file.
+        self.set_parameters: dict[int, tuple[Any, bool]] = {}
 
         #: Keys are parameter indices and values are dict parameter sources to merge
         #: with existing source of that parameter.
-        self.update_param_sources: Dict[int, Dict] = None
+        self.update_param_sources: dict[int, ParamSource] = {}
         #: Keys are indices of loops, values are descriptions of what to update.
-        self.update_loop_indices: Dict[int, Dict[str, int]] = None
+        self.update_loop_indices: dict[int, dict[str, int]] = {}
         #: Keys are indices of loops, values are number of iterations.
-        self.update_loop_num_iters: Dict[int, int] = None
+        self.update_loop_num_iters: dict[int, list[list[list[int] | int]]] = {}
         #: Keys are indices of loops, values are list of parent names.
-        self.update_loop_parents: Dict[int, List[str]] = None
+        self.update_loop_parents: dict[int, list[str]] = {}
+
+        self.update_iter_data_idx: dict[int, DataIndex] = {}
+        self.update_run_data_idx: dict[int, DataIndex] = {}
 
         self.reset(is_init=True)  # set up initial data structures
 
@@ -106,43 +176,99 @@ class PendingChanges:
             or bool(self.add_elem_IDs)
             or bool(self.add_elem_iter_IDs)
             or bool(self.add_elem_iter_EAR_IDs)
-            or bool(self.add_submission_parts)
+            or bool(self.update_at_submit_metadata)
             or bool(self.add_parameters)
             or bool(self.add_files)
             or bool(self.add_template_components)
             or bool(self.add_element_sets)
             or bool(self.set_EARs_initialised)
-            or bool(self.set_EAR_submission_indices)
+            or bool(self.set_EAR_submission_data)
             or bool(self.set_EAR_starts)
             or bool(self.set_EAR_ends)
             or bool(self.set_EAR_skips)
+            or bool(self.set_run_dirs)
             or bool(self.set_js_metadata)
             or bool(self.set_parameters)
             or bool(self.update_param_sources)
             or bool(self.update_loop_indices)
             or bool(self.update_loop_num_iters)
             or bool(self.update_loop_parents)
+            or bool(self.update_iter_data_idx)
+            or bool(self.update_run_data_idx)
         )
 
-    def where_pending(self) -> List[str]:
+    def where_pending(self) -> list[str]:
         """
         Get the list of items for which there is some outstanding pending items.
         """
-        return [
-            k
-            for k, v in self.__dict__.items()
-            if k not in ("app", "store", "resource_map") and bool(v)
-        ]
+        excluded = {"app", "store", "resource_map"}
+        return [k for k, v in self.__dict__.items() if k not in excluded and bool(v)]
 
     @property
-    def logger(self):
+    def logger(self) -> Logger:
         """
         The logger.
         """
-        return self.app.persistence_logger
+        return self._app.persistence_logger
+
+    def commits_data(*data_list: str):
+        """Decorator that wraps `PendingChanges.commit_*` methods with arguments listing
+        which `PendingChanges` attributes must have non-trivial data in them for the method's
+        invocation to be required.
+
+        Notes
+        -----
+        This essentially provides a mapping between `PendingChanges` attributes and
+        `commit_*` methods. This allows us to only open the resources that need updating
+        in `PendingChanges.commit_all`.
+
+        We use a decorator rather than an explicitly declaring the map in
+        `_commit_method_data_map` to make the mapping obvious near the commit methods, and
+        hopefully avoid us forgetting to update `_commit_method_data_map` when we modify
+        or add commit methods in future!
+
+        """
+
+        def decorator(func: Callable[P, T]) -> Callable[P, T]:
+
+            _commit_method_data_map[func.__name__].extend(data_list)
+
+            @wraps(func)
+            def inner(*args, **kwargs) -> T:
+                return func(*args, **kwargs)
+
+            return inner
+
+        return decorator
+
+    def get_pending_resource_map_groups(self) -> dict[tuple[str, ...], list[str]]:
+        """Retrive resource map groups, where values are filtered to include only those
+        commit methods that must be invoked, due to pending data associated with those
+        methods.
+
+        Notes
+        -----
+        This method allows us to open only those resources that need to be updated, given
+        the state of pending data.
+        """
+
+        where_pending = self.where_pending()
+        pending_groups = {}
+        for res_names, methods in self.resource_map.groups.items():
+            req_methods = [
+                meth_i
+                for meth_i in methods
+                if any(
+                    dat_j in where_pending for dat_j in _commit_method_data_map[meth_i]
+                )
+            ]
+            if req_methods:
+                pending_groups[res_names] = req_methods
+
+        return pending_groups
 
     @TimeIt.decorator
-    def commit_all(self):
+    def commit_all(self) -> None:
         """Commit all pending changes to disk."""
         self.logger.info(f"committing all pending changes: {self.where_pending()}")
 
@@ -150,23 +276,25 @@ class PendingChanges:
             self.logger.debug("commit: no pending changes to commit.")
             return
 
-        for resources, methods in self.resource_map.groups.items():
+        for resources, methods in self.get_pending_resource_map_groups().items():
             # for each resource, enter `using_resource` context manager in "update" mode:
             with contextlib.ExitStack() as stack:
                 for res in resources:
-                    # TODO: only enter required resources!
-                    stack.enter_context(self.store.using_resource(res, "update"))
+                    stack.enter_context(
+                        self.store.using_resource(res, "update")  # type: ignore[call-overload]
+                    )
                 for meth in methods:
                     getattr(self, meth)()
 
         assert not (self)
 
     @TimeIt.decorator
+    @commits_data("add_tasks")
     def commit_tasks(self) -> None:
         """Commit pending tasks to disk."""
         if self.add_tasks:
             tasks = self.store.get_tasks_by_IDs(self.add_tasks)
-            task_ids = list(self.add_tasks.keys())
+            task_ids = set(self.add_tasks)
             self.logger.debug(f"commit: adding pending tasks with IDs: {task_ids!r}")
             self.store._append_tasks(tasks)
             self.store.num_tasks_cache = None  # invalidate cache
@@ -177,12 +305,13 @@ class PendingChanges:
         self._clear_add_tasks()
 
     @TimeIt.decorator
+    @commits_data("add_loops")
     def commit_loops(self) -> None:
         """Commit pending loops to disk."""
         if self.add_loops:
             # retrieve pending loops, including pending changes to num_added_iterations:
             loops = self.store.get_loops_by_IDs(self.add_loops)
-            loop_ids = list(self.add_loops.keys())
+            loop_ids = set(self.add_loops)
             self.logger.debug(f"commit: adding pending loops with indices {loop_ids!r}")
             self.store._append_loops(loops)
 
@@ -198,12 +327,15 @@ class PendingChanges:
         self._clear_add_loops()
 
     @TimeIt.decorator
+    @commits_data("add_submissions")
     def commit_submissions(self) -> None:
         """Commit pending submissions to disk."""
         if self.add_submissions:
             # retrieve pending submissions:
-            subs = self.store.get_submissions_by_ID(self.add_submissions)
-            sub_ids = list(self.add_submissions.keys())
+            subs = self.store.get_submissions_by_ID(
+                self.add_submissions
+            )  # TODO: I think this just returns add_submissions?
+            sub_ids = set(self.add_submissions)
             self.logger.debug(
                 f"commit: adding pending submissions with indices {sub_ids!r}"
             )
@@ -211,16 +343,18 @@ class PendingChanges:
         self._clear_add_submissions()
 
     @TimeIt.decorator
-    def commit_submission_parts(self) -> None:
+    @commits_data("update_at_submit_metadata")
+    def commit_at_submit_metadata(self) -> None:
         """
-        Commit pending submission parts to disk.
+        Commit to disk pending at-submit-time metadata, including submission parts.
         """
-        if self.add_submission_parts:
-            self.logger.debug(f"commit: adding pending submission parts")
-            self.store._append_submission_parts(self.add_submission_parts)
-        self._clear_add_submission_parts()
+        if self.update_at_submit_metadata:
+            self.logger.debug("commit: adding pending at-submit metadata")
+            self.store._update_at_submit_metadata(self.update_at_submit_metadata)
+        self._clear_at_submit_metadata()
 
     @TimeIt.decorator
+    @commits_data("add_elem_IDs")
     def commit_elem_IDs(self) -> None:
         """
         Commit pending element ID updates to disk.
@@ -235,13 +369,14 @@ class PendingChanges:
         self._clear_add_elem_IDs()
 
     @TimeIt.decorator
+    @commits_data("add_elements")
     def commit_elements(self) -> None:
         """
         Commit pending elements to disk.
         """
         if self.add_elements:
             elems = self.store.get_elements(self.add_elements)
-            elem_ids = list(self.add_elements.keys())
+            elem_ids = set(self.add_elements)
             self.logger.debug(f"commit: adding pending elements with IDs: {elem_ids!r}")
             self.store._append_elements(elems)
             # pending iter IDs that belong to pending elements are now committed:
@@ -251,17 +386,19 @@ class PendingChanges:
         self._clear_add_elements()
 
     @TimeIt.decorator
+    @commits_data("add_element_sets")
     def commit_element_sets(self) -> None:
         """
         Commit pending element sets to disk.
         """
         # TODO: could be batched up?
         for task_id, es_js in self.add_element_sets.items():
-            self.logger.debug(f"commit: adding pending element sets.")
+            self.logger.debug("commit: adding pending element sets.")
             self.store._append_element_sets(task_id, es_js)
         self._clear_add_element_sets()
 
     @TimeIt.decorator
+    @commits_data("add_elem_iter_IDs")
     def commit_elem_iter_IDs(self) -> None:
         """
         Commit pending element iteration ID updates to disk.
@@ -277,21 +414,27 @@ class PendingChanges:
         self._clear_add_elem_iter_IDs()
 
     @TimeIt.decorator
+    @commits_data("add_elem_iters")
     def commit_elem_iters(self) -> None:
         """
         Commit pending element iterations to disk.
         """
         if self.add_elem_iters:
-            iters = self.store.get_element_iterations(self.add_elem_iters.keys())
-            iter_ids = list(self.add_elem_iters.keys())
+            iters = self.store.get_element_iterations(self.add_elem_iters)
+            iter_ids = set(self.add_elem_iters)
             self.logger.debug(
                 f"commit: adding pending element iterations with IDs: {iter_ids!r}"
             )
             self.store._append_elem_iters(iters)
             # pending EAR IDs that belong to pending iters are now committed:
-            self.add_elem_iter_EAR_IDs = {
-                k: v for k, v in self.add_elem_iter_EAR_IDs.items() if k not in iter_ids
-            }
+            add_elem_iter_EAR_IDs_cur = copy.deepcopy(self.add_elem_iter_EAR_IDs)
+            self._clear_add_elem_iter_EAR_IDs()  # reset to empty nested defaultdict
+            for iter_id, all_run_IDs in add_elem_iter_EAR_IDs_cur.items():
+                # only re-assign iter_IDs that have not been comitted above:
+                if iter_id not in iter_ids:
+                    for act_idx, run_IDs in all_run_IDs.items():
+                        self.add_elem_iter_EAR_IDs[iter_id][act_idx].extend(run_IDs)
+
             # pending EARs_initialised that belong to pending iters are now committed:
             self.set_EARs_initialised = [
                 i for i in self.set_EARs_initialised if i not in iter_ids
@@ -299,6 +442,7 @@ class PendingChanges:
         self._clear_add_elem_iters()
 
     @TimeIt.decorator
+    @commits_data("add_elem_iter_EAR_IDs")
     def commit_elem_iter_EAR_IDs(self) -> None:
         """
         Commit pending element action run ID updates to disk.
@@ -315,24 +459,25 @@ class PendingChanges:
         self._clear_add_elem_iter_EAR_IDs()
 
     @TimeIt.decorator
+    @commits_data("add_EARs")
     def commit_EARs(self) -> None:
         """
         Commit pending element action runs to disk.
         """
         if self.add_EARs:
             EARs = self.store.get_EARs(self.add_EARs)
-            EAR_ids = list(self.add_EARs.keys())
+            EAR_ids = list(self.add_EARs)
             self.logger.debug(f"commit: adding pending EARs with IDs: {EAR_ids!r}")
             self.store._append_EARs(EARs)
             self.store.num_EARs_cache = None  # invalidate cache
             # pending start/end times/snapshots, submission indices, and skips that belong
             # to pending EARs are now committed (accounted for in `get_EARs` above):
-            self.set_EAR_submission_indices = {
-                k: v
-                for k, v in self.set_EAR_submission_indices.items()
-                if k not in EAR_ids
+            self.set_EAR_submission_data = {
+                k: v for k, v in self.set_EAR_submission_data.items() if k not in EAR_ids
+            }
+            self.set_EAR_skips = {
+                k: v for k, v in self.set_EAR_skips.items() if k not in EAR_ids
             }
-            self.set_EAR_skips = [i for i in self.set_EAR_skips if i not in EAR_ids]
             self.set_EAR_starts = {
                 k: v for k, v in self.set_EAR_starts.items() if k not in EAR_ids
             }
@@ -343,6 +488,17 @@ class PendingChanges:
         self._clear_add_EARs()
 
     @TimeIt.decorator
+    @commits_data("set_run_dirs")
+    def commit_set_run_dirs(self) -> None:
+        """
+        Commit pending run directory indices.
+        """
+        for run_dir_arr, run_idx in self.set_run_dirs:
+            self.store._set_run_dirs(run_dir_arr, run_idx)
+        self._clear_set_run_dirs()
+
+    @TimeIt.decorator
+    @commits_data("set_EARs_initialised")
     def commit_EARs_initialised(self) -> None:
         """
         Commit pending element action run init state updates to disk.
@@ -360,63 +516,71 @@ class PendingChanges:
         self._clear_set_EARs_initialised()
 
     @TimeIt.decorator
+    @commits_data("set_EAR_submission_data")
     def commit_EAR_submission_indices(self) -> None:
         """
         Commit pending element action run submission index updates to disk.
         """
-        if self.set_EAR_submission_indices:
+        if self.set_EAR_submission_data:
             self.logger.debug(
-                f"commit: updating submission indices: "
-                f"{self.set_EAR_submission_indices!r}."
+                f"commit: updating submission data: {self.set_EAR_submission_data!r}."
             )
-            self.store._update_EAR_submission_indices(self.set_EAR_submission_indices)
-            for EAR_ID_i in self.set_EAR_submission_indices.keys():
+            self.store._update_EAR_submission_data(self.set_EAR_submission_data)
+            for EAR_ID_i in self.set_EAR_submission_data:
                 self.store.EAR_cache.pop(EAR_ID_i, None)  # invalidate cache
-            self._clear_set_EAR_submission_indices()
+            self._clear_EAR_submission_data()
 
     @TimeIt.decorator
+    @commits_data("set_EAR_starts")
     def commit_EAR_starts(self) -> None:
         """
         Commit pending element action run start information to disk.
         """
-        # TODO: could be batched up?
-        for EAR_id, (time, snap, hostname) in self.set_EAR_starts.items():
+        updates = self.set_EAR_starts
+        if updates:
             self.logger.debug(
-                f"commit: adding pending start time ({time!r}), run hostname "
-                f"({hostname!r}), and directory snapshot to EAR ID {EAR_id!r}."
+                f"commit: registering {len(updates)} run(s) as started: "
+                f"{shorten_list_str(updates)}."
             )
-            self.store._update_EAR_start(EAR_id, time, snap, hostname)
-            self.store.EAR_cache.pop(EAR_id, None)  # invalidate cache
+            self.store._update_EAR_start(updates)
+            for run_id in updates:
+                self.store.EAR_cache.pop(run_id, None)  # invalidate cache
         self._clear_set_EAR_starts()
 
     @TimeIt.decorator
+    @commits_data("set_EAR_ends")
     def commit_EAR_ends(self) -> None:
         """
         Commit pending element action run finish information to disk.
         """
-        # TODO: could be batched up?
-        for EAR_id, (time, snap, ext, suc) in self.set_EAR_ends.items():
+        updates = self.set_EAR_ends
+        if updates:
             self.logger.debug(
-                f"commit: adding pending end time ({time!r}), directory snapshot, "
-                f"exit code ({ext!r}), and success status {suc!r} to EAR ID {EAR_id!r}."
+                f"commit: registering {len(updates)} run(s) as ended: "
+                f"{shorten_list_str(updates)}, with exit codes: "
+                f"{shorten_list_str([i[2] for i in updates.values()])}."
             )
-            self.store._update_EAR_end(EAR_id, time, snap, ext, suc)
-            self.store.EAR_cache.pop(EAR_id, None)  # invalidate cache
+            self.store._update_EAR_end(updates)
+            for run_id in updates:
+                self.store.EAR_cache.pop(run_id, None)  # invalidate cache
         self._clear_set_EAR_ends()
 
     @TimeIt.decorator
+    @commits_data("set_EAR_skips")
     def commit_EAR_skips(self) -> None:
         """
         Commit pending element action skip flags to disk.
         """
-        # TODO: could be batched up?
-        for EAR_id in self.set_EAR_skips:
-            self.logger.debug(f"commit: setting EAR ID {EAR_id!r} as skipped.")
-            self.store._update_EAR_skip(EAR_id)
-            self.store.EAR_cache.pop(EAR_id, None)  # invalidate cache
+        updates = self.set_EAR_skips
+        if updates:
+            self.logger.debug(f"commit: setting {len(updates)} run IDs as skipped.")
+            self.store._update_EAR_skip(updates)
+            for run_ID in updates:
+                self.store.EAR_cache.pop(run_ID, None)  # invalidate cache
         self._clear_set_EAR_skips()
 
     @TimeIt.decorator
+    @commits_data("set_js_metadata")
     def commit_js_metadata(self) -> None:
         """
         Commit pending jobscript metadata changes to disk.
@@ -429,47 +593,50 @@ class PendingChanges:
         self._clear_set_js_metadata()
 
     @TimeIt.decorator
+    @commits_data("add_parameters", "set_parameters")
     def commit_parameters(self) -> None:
         """Make pending parameters persistent."""
         if self.add_parameters:
             params = self.store.get_parameters(self.add_parameters)
-            param_ids = list(self.add_parameters.keys())
+            param_ids = list(self.add_parameters)
             self.logger.debug(f"commit: adding pending parameters IDs: {param_ids!r}")
             self.store._append_parameters(params)
         self._clear_add_parameters()
 
         if self.set_parameters:
-            param_ids = list(self.set_parameters.keys())
+            param_ids = list(self.set_parameters)
             self.logger.debug(f"commit: setting values of parameter IDs {param_ids!r}.")
             self.store._set_parameter_values(self.set_parameters)
             for id_i in param_ids:
                 self.store.parameter_cache.pop(id_i, None)
-
         self._clear_set_parameters()
 
     @TimeIt.decorator
+    @commits_data("add_files")
     def commit_files(self) -> None:
         """Add pending files to the files directory."""
         if self.add_files:
-            self.logger.debug(f"commit: adding pending files to the files directory.")
+            self.logger.debug("commit: adding pending files to the files directory.")
             self.store._append_files(self.add_files)
         self._clear_add_files()
 
     @TimeIt.decorator
+    @commits_data("add_template_components")
     def commit_template_components(self) -> None:
         """
         Commit pending template components to disk.
         """
         if self.add_template_components:
-            self.logger.debug(f"commit: adding template components.")
+            self.logger.debug("commit: adding template components.")
             self.store._update_template_components(self.store.get_template_components())
         self._clear_add_template_components()
 
     @TimeIt.decorator
+    @commits_data("update_param_sources")
     def commit_param_sources(self) -> None:
         """Make pending changes to parameter sources persistent."""
         if self.update_param_sources:
-            param_ids = list(self.update_param_sources.keys())
+            param_ids = list(self.update_param_sources)
             self.logger.debug(f"commit: updating sources of parameter IDs {param_ids!r}.")
             self.store._update_parameter_sources(self.update_param_sources)
             for id_i in param_ids:
@@ -477,19 +644,21 @@ class PendingChanges:
             self._clear_update_param_sources()
 
     @TimeIt.decorator
+    @commits_data("update_loop_indices")
     def commit_loop_indices(self) -> None:
         """Make pending update to element iteration loop indices persistent."""
-        # TODO: batch up
-        for iter_ID, loop_idx in self.update_loop_indices.items():
+        updates = self.update_loop_indices
+        if updates:
             self.logger.debug(
-                f"commit: updating loop indices of iteration ID {iter_ID!r} with "
-                f"{loop_idx!r}."
+                f"commit: updating loop indices of {len(updates)} iteration(s)."
             )
-            self.store._update_loop_index(iter_ID, loop_idx)
-            self.store.element_iter_cache.pop(iter_ID, None)  # invalidate cache
+            self.store._update_loop_index(updates)
+            for iter_ID in updates:
+                self.store.element_iter_cache.pop(iter_ID, None)  # invalidate cache
         self._clear_update_loop_indices()
 
     @TimeIt.decorator
+    @commits_data("update_loop_num_iters")
     def commit_loop_num_iters(self) -> None:
         """Make pending update to the number of loop iterations."""
         for index, num_iters in self.update_loop_num_iters.items():
@@ -500,6 +669,7 @@ class PendingChanges:
         self._clear_update_loop_num_iters()
 
     @TimeIt.decorator
+    @commits_data("update_loop_parents")
     def commit_loop_parents(self) -> None:
         """Make pending update to additional loop parents."""
         for index, parents in self.update_loop_parents.items():
@@ -507,82 +677,107 @@ class PendingChanges:
             self.store._update_loop_parents(index, parents)
         self._clear_update_loop_parents()
 
-    def _clear_add_tasks(self):
+    @TimeIt.decorator
+    @commits_data("update_iter_data_idx")
+    def commit_iter_data_idx(self) -> None:
+        if self.update_iter_data_idx:
+            self.store._update_iter_data_indices(self.update_iter_data_idx)
+            self._clear_update_iter_data_idx()
+
+    @TimeIt.decorator
+    @commits_data("update_run_data_idx")
+    def commit_run_data_idx(self) -> None:
+        if self.update_run_data_idx:
+            self.store._update_run_data_indices(self.update_run_data_idx)
+            self._clear_update_run_data_idx()
+
+    def _clear_add_tasks(self) -> None:
         self.add_tasks = {}
 
-    def _clear_add_loops(self):
+    def _clear_add_loops(self) -> None:
         self.add_loops = {}
 
-    def _clear_add_submissions(self):
+    def _clear_add_submissions(self) -> None:
         self.add_submissions = {}
 
-    def _clear_add_submission_parts(self):
-        self.add_submission_parts = defaultdict(dict)
+    def _clear_at_submit_metadata(self) -> None:
+        self.update_at_submit_metadata = defaultdict(
+            lambda: {i: None for i in SUBMISSION_SUBMIT_TIME_KEYS}
+        )
 
-    def _clear_add_elements(self):
+    def _clear_add_elements(self) -> None:
         self.add_elements = {}
 
-    def _clear_add_element_sets(self):
+    def _clear_add_element_sets(self) -> None:
         self.add_element_sets = defaultdict(list)
 
-    def _clear_add_elem_iters(self):
+    def _clear_add_elem_iters(self) -> None:
         self.add_elem_iters = {}
 
-    def _clear_add_EARs(self):
+    def _clear_add_EARs(self) -> None:
         self.add_EARs = {}
 
-    def _clear_add_elem_IDs(self):
+    def _clear_set_run_dirs(self):
+        self.set_run_dirs = []
+
+    def _clear_add_elem_IDs(self) -> None:
         self.add_elem_IDs = defaultdict(list)
 
-    def _clear_add_elem_iter_IDs(self):
+    def _clear_add_elem_iter_IDs(self) -> None:
         self.add_elem_iter_IDs = defaultdict(list)
 
-    def _clear_add_elem_iter_EAR_IDs(self):
+    def _clear_add_elem_iter_EAR_IDs(self) -> None:
         self.add_elem_iter_EAR_IDs = defaultdict(lambda: defaultdict(list))
 
-    def _clear_set_EARs_initialised(self):
+    def _clear_set_EARs_initialised(self) -> None:
         self.set_EARs_initialised = []
 
-    def _clear_set_EAR_submission_indices(self):
-        self.set_EAR_submission_indices = {}
+    def _clear_EAR_submission_data(self) -> None:
+        self.set_EAR_submission_data = {}
 
-    def _clear_set_EAR_starts(self):
+    def _clear_set_EAR_starts(self) -> None:
         self.set_EAR_starts = {}
 
-    def _clear_set_EAR_ends(self):
+    def _clear_set_EAR_ends(self) -> None:
         self.set_EAR_ends = {}
 
-    def _clear_set_EAR_skips(self):
-        self.set_EAR_skips = []
+    def _clear_set_EAR_skips(self) -> None:
+        self.set_EAR_skips = {}
 
-    def _clear_set_js_metadata(self):
+    def _clear_set_js_metadata(self) -> None:
         self.set_js_metadata = defaultdict(lambda: defaultdict(dict))
 
-    def _clear_add_parameters(self):
+    def _clear_add_parameters(self) -> None:
         self.add_parameters = {}
 
-    def _clear_add_files(self):
+    def _clear_add_files(self) -> None:
         self.add_files = []
 
-    def _clear_add_template_components(self):
+    def _clear_add_template_components(self) -> None:
         self.add_template_components = defaultdict(dict)
 
-    def _clear_set_parameters(self):
+    def _clear_set_parameters(self) -> None:
         self.set_parameters = {}
 
-    def _clear_update_param_sources(self):
+    def _clear_update_param_sources(self) -> None:
         self.update_param_sources = {}
 
-    def _clear_update_loop_indices(self):
+    def _clear_update_loop_indices(self) -> None:
         self.update_loop_indices = defaultdict(dict)
 
-    def _clear_update_loop_num_iters(self):
+    def _clear_update_loop_num_iters(self) -> None:
         self.update_loop_num_iters = {}
 
-    def _clear_update_loop_parents(self):
+    def _clear_update_loop_parents(self) -> None:
         self.update_loop_parents = {}
 
-    def reset(self, is_init=False) -> None:
+    def _clear_update_iter_data_idx(self):
+        self.update_iter_data_idx = defaultdict(dict)
+
+    def _clear_update_run_data_idx(self):
+        self.update_run_data_idx = defaultdict(dict)
+
+    def reset(self, is_init: bool = False) -> None:
         """Clear all pending data and prepare to accept new pending data."""
 
         if not is_init and not self:
@@ -595,11 +790,12 @@ class PendingChanges:
         self._clear_add_tasks()
         self._clear_add_loops()
         self._clear_add_submissions()
-        self._clear_add_submission_parts()
+        self._clear_at_submit_metadata()
         self._clear_add_elements()
         self._clear_add_element_sets()
         self._clear_add_elem_iters()
         self._clear_add_EARs()
+        self._clear_set_run_dirs()
 
         self._clear_set_EARs_initialised()
         self._clear_add_elem_IDs()
@@ -610,7 +806,7 @@ class PendingChanges:
         self._clear_add_files()
         self._clear_add_template_components()
 
-        self._clear_set_EAR_submission_indices()
+        self._clear_EAR_submission_data()
         self._clear_set_EAR_starts()
         self._clear_set_EAR_ends()
         self._clear_set_EAR_skips()
@@ -622,6 +818,8 @@ class PendingChanges:
         self._clear_update_loop_indices()
         self._clear_update_loop_num_iters()
         self._clear_update_loop_parents()
+        self._clear_update_iter_data_idx()
+        self._clear_update_run_data_idx()
 
 
 @dataclass
@@ -640,65 +838,74 @@ class CommitResourceMap:
     """
 
     #: Resources for :py:meth:`~.PendingChanges.commit_tasks`.
-    commit_tasks: Optional[Tuple[str]] = tuple()
+    commit_tasks: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_loops`.
-    commit_loops: Optional[Tuple[str]] = tuple()
+    commit_loops: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_submissions`.
-    commit_submissions: Optional[Tuple[str]] = tuple()
-    #: Resources for :py:meth:`~.PendingChanges.commit_submission_parts`.
-    commit_submission_parts: Optional[Tuple[str]] = tuple()
+    commit_submissions: tuple[str, ...] | None = tuple()
+    #: Resources for :py:meth:`~.PendingChanges.commit_at_submit_metadata`.
+    commit_at_submit_metadata: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_elem_IDs`.
-    commit_elem_IDs: Optional[Tuple[str]] = tuple()
+    commit_elem_IDs: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_elements`.
-    commit_elements: Optional[Tuple[str]] = tuple()
+    commit_elements: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_element_sets`.
-    commit_element_sets: Optional[Tuple[str]] = tuple()
+    commit_element_sets: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_elem_iter_IDs`.
-    commit_elem_iter_IDs: Optional[Tuple[str]] = tuple()
+    commit_elem_iter_IDs: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_elem_iters`.
-    commit_elem_iters: Optional[Tuple[str]] = tuple()
+    commit_elem_iters: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_elem_iter_EAR_IDs`.
-    commit_elem_iter_EAR_IDs: Optional[Tuple[str]] = tuple()
+    commit_elem_iter_EAR_IDs: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_EARs_initialised`.
-    commit_EARs_initialised: Optional[Tuple[str]] = tuple()
+    commit_EARs_initialised: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_EARs`.
-    commit_EARs: Optional[Tuple[str]] = tuple()
+    commit_EARs: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_EAR_submission_indices`.
-    commit_EAR_submission_indices: Optional[Tuple[str]] = tuple()
+    commit_EAR_submission_indices: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_EAR_skips`.
-    commit_EAR_skips: Optional[Tuple[str]] = tuple()
+    commit_EAR_skips: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_EAR_starts`.
-    commit_EAR_starts: Optional[Tuple[str]] = tuple()
+    commit_EAR_starts: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_EAR_ends`.
-    commit_EAR_ends: Optional[Tuple[str]] = tuple()
+    commit_EAR_ends: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_js_metadata`.
-    commit_js_metadata: Optional[Tuple[str]] = tuple()
+    commit_js_metadata: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_parameters`.
-    commit_parameters: Optional[Tuple[str]] = tuple()
+    commit_parameters: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_files`.
-    commit_files: Optional[Tuple[str]] = tuple()
+    commit_files: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_template_components`.
-    commit_template_components: Optional[Tuple[str]] = tuple()
+    commit_template_components: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_param_sources`.
-    commit_param_sources: Optional[Tuple[str]] = tuple()
+    commit_param_sources: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_loop_indices`.
-    commit_loop_indices: Optional[Tuple[str]] = tuple()
+    commit_loop_indices: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_loop_num_iters`.
-    commit_loop_num_iters: Optional[Tuple[str]] = tuple()
+    commit_loop_num_iters: tuple[str, ...] | None = tuple()
     #: Resources for :py:meth:`~.PendingChanges.commit_loop_parents`.
-    commit_loop_parents: Optional[Tuple[str]] = tuple()
+    commit_loop_parents: tuple[str, ...] | None = tuple()
+    #: Resources for :py:meth:`~.PendingChanges.commit_set_run_dirs`.
+    commit_set_run_dirs: tuple[str, ...] | None = tuple()
+    #: Resources for :py:meth:`~.PendingChanges.commit_iter_data_idx`.
+    commit_iter_data_idx: tuple[str, ...] | None = tuple()
+    #: Resources for :py:meth:`~.PendingChanges.commit_run_data_idx`.
+    commit_run_data_idx: tuple[str, ...] | None = tuple()
+
     #: A dict whose keys are tuples of resource labels and whose values are lists
     #: of :py:class:`PendingChanges` commit method names that require those resources.
     #:
     #: This grouping allows us to batch up commit methods by resource requirements,
     #: which in turn means we can potentially minimise, e.g., the number of network
     #: requests.
-    groups: Dict[Tuple[str], List[str]] = field(init=False, repr=False, compare=False)
+    groups: Mapping[tuple[str, ...], Sequence[str]] = field(
+        init=False, repr=False, compare=False
+    )
 
     def __post_init__(self):
         self.groups = self._group_by_resource()
 
-    def _group_by_resource(self) -> Dict[Tuple[str], List[str]]:
+    def _group_by_resource(self) -> Mapping[tuple[str, ...], Sequence[str]]:
         """
         Get a dict whose keys are tuples of resource labels and whose values are
         lists of :py:class:`PendingChanges` commit method names that require those
@@ -708,8 +915,9 @@ class CommitResourceMap:
         which in turn means we can potentially minimise e.g. the number of network
         requests.
         """
-        groups = {}
-        cur_res_group = None
+        groups: dict[tuple[str, ...], list[str]] = {}
+        # The dicts are pretending to be insertion-ordered sets
+        cur_res_group: tuple[dict[str, None], list[str]] | None = None
         for fld in fields(self):
             if not fld.name.startswith("commit_"):
                 continue
@@ -718,28 +926,23 @@ class CommitResourceMap:
             if not cur_res_group:
                 # start a new resource group: a mapping between resource labels and the
                 # commit methods that require those resources:
-                cur_res_group = [list(res_labels), [fld.name]]
+                cur_res_group = (dict.fromkeys(res_labels), [fld.name])
 
             elif not res_labels or set(res_labels).intersection(cur_res_group[0]):
                 # there is some overlap between resource labels required in the current
                 # group and this commit method, so we merge resource labels and add the
                 # new commit method:
-                cur_res_group[0] = list(set(cur_res_group[0] + list(res_labels)))
+                cur_res_group[0].update(dict.fromkeys(res_labels))
                 cur_res_group[1].append(fld.name)
 
             else:
                 # no overlap between resource labels required in the current group and
                 # those required by this commit method, so append the current group, and
                 # start a new group for this commit method:
-                if tuple(cur_res_group[0]) not in groups:
-                    groups[tuple(cur_res_group[0])] = []
-                groups[tuple(cur_res_group[0])].extend(cur_res_group[1])
-                cur_res_group = [list(res_labels), [fld.name]]
+                groups.setdefault(tuple(cur_res_group[0]), []).extend(cur_res_group[1])
+                cur_res_group = (dict.fromkeys(res_labels), [fld.name])
 
         if cur_res_group:
-            if tuple(cur_res_group[0]) not in groups:
-                groups[tuple(cur_res_group[0])] = []
-
-            groups[tuple(cur_res_group[0])].extend(cur_res_group[1])
+            groups.setdefault(tuple(cur_res_group[0]), []).extend(cur_res_group[1])
 
         return groups