hpcflow-new2 0.2.0a176__py3-none-any.whl → 0.2.0a177__py3-none-any.whl

hpcflow/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0a176"
+__version__ = "0.2.0a177"

hpcflow/sdk/core/cache.py

@@ -0,0 +1,142 @@
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Set, Dict
+
+from hpcflow.sdk.log import TimeIt
+
+
+@dataclass
+class DependencyCache:
+    """Class to bulk-retrieve dependencies between elements, iterations, and runs."""
+
+    run_dependencies: Dict[int, Set]
+    run_dependents: Dict[int, Set]
+    iter_run_dependencies: Dict[int, Set]
+    iter_iter_dependencies: Dict[int, Set]
+    elem_iter_dependencies: Dict[int, Set]
+    elem_elem_dependencies: Dict[int, Set]
+    elem_elem_dependents: Dict[int, Set]
+    elem_elem_dependents_rec: Dict[int, Set]
+
+    elements: Dict
+    iterations: Dict
+
+    @classmethod
+    @TimeIt.decorator
+    def build(cls, workflow):
+        num_iters = workflow.num_element_iterations
+        num_elems = workflow.num_elements
+        num_runs = workflow.num_EARs
+
+        all_store_runs = workflow._store.get_EARs(list(range(num_runs)))
+        all_store_iters = workflow._store.get_element_iterations(list(range(num_iters)))
+        all_store_elements = workflow._store.get_elements(list(range(num_elems)))
+        all_param_sources = workflow.get_all_parameter_sources()
+        all_data_idx = [
+            {
+                k: v if isinstance(v, list) else [v]
+                for k, v in i.data_idx.items()
+                if k not in ("repeats.",)
+            }
+            for i in all_store_runs
+        ]
+
+        # run dependencies and dependents
+        run_dependencies = {}
+        run_dependents = defaultdict(set)
+        for idx, i in enumerate(all_data_idx):
+            run_i_sources = set()
+            for j in i.values():
+                for k in j:
+                    run_k = all_param_sources[k].get("EAR_ID")
+                    if run_k is not None and run_k != idx:
+                        run_i_sources.add(run_k)
+            run_dependencies[idx] = run_i_sources
+            for m in run_i_sources:
+                run_dependents[m].add(idx)
+
+        # add missing:
+        for k in range(num_runs):
+            run_dependents[k]
+
+        run_dependents = dict(run_dependents)
+
+        # iteration dependencies
+        all_iter_run_IDs = {
+            i.id_: [k for j in i.EAR_IDs.values() for k in j] for i in all_store_iters
+        }
+        # for each iteration, which runs does it depend on?
+        iter_run_dependencies = {
+            k: set(j for i in v for j in run_dependencies[i])
+            for k, v in all_iter_run_IDs.items()
+        }
+
+        # for each run, which iteration does it belong to?
+        all_run_iter_IDs = {}
+        for iter_ID, run_IDs in all_iter_run_IDs.items():
+            for run_ID in run_IDs:
+                all_run_iter_IDs[run_ID] = iter_ID
+
+        # for each iteration, which iterations does it depend on?
+        iter_iter_dependencies = {
+            k: set(all_run_iter_IDs[i] for i in v)
+            for k, v in iter_run_dependencies.items()
+        }
+
+        all_elem_iter_IDs = {i.id_: i.iteration_IDs for i in all_store_elements}
+
+        elem_iter_dependencies = {
+            k: set(j for i in v for j in iter_iter_dependencies[i])
+            for k, v in all_elem_iter_IDs.items()
+        }
+
+        # for each iteration, which element does it belong to?
+        all_iter_elem_IDs = {}
+        for elem_ID, iter_IDs in all_elem_iter_IDs.items():
+            for iter_ID in iter_IDs:
+                all_iter_elem_IDs[iter_ID] = elem_ID
+
+        # element dependencies
+        elem_elem_dependencies = {
+            k: set(all_iter_elem_IDs[i] for i in v)
+            for k, v in elem_iter_dependencies.items()
+        }
+
+        # for each element, which elements depend on it (directly)?
+        elem_elem_dependents = defaultdict(set)
+        for k, v in elem_elem_dependencies.items():
+            for i in v:
+                elem_elem_dependents[i].add(k)
+
+        # for each element, which elements depend on it (recursively)?
+        elem_elem_dependents_rec = defaultdict(set)
+        for k in list(elem_elem_dependents):
+            for i in elem_elem_dependents[k]:
+                elem_elem_dependents_rec[k].add(i)
+                elem_elem_dependents_rec[k].update(
+                    {m for m in elem_elem_dependents[i] if m != k}
+                )
+
+        # add missing keys:
+        for k in range(num_elems):
+            elem_elem_dependents[k]
+            elem_elem_dependents_rec[k]
+
+        elem_elem_dependents = dict(elem_elem_dependents)
+        elem_elem_dependents_rec = dict(elem_elem_dependents_rec)
+
+        elements = workflow.get_all_elements()
+        iterations = workflow.get_all_element_iterations()
+
+        return cls(
+            run_dependencies=run_dependencies,
+            run_dependents=run_dependents,
+            iter_run_dependencies=iter_run_dependencies,
+            iter_iter_dependencies=iter_iter_dependencies,
+            elem_iter_dependencies=elem_iter_dependencies,
+            elem_elem_dependencies=elem_elem_dependencies,
+            elem_elem_dependents=elem_elem_dependents,
+            elem_elem_dependents_rec=elem_elem_dependents_rec,
+            elements=elements,
+            iterations=iterations,
+        )

hpcflow/sdk/core/element.py

@@ -675,6 +675,7 @@ class ElementIteration:
             default=default,
         )
 
+    @TimeIt.decorator
     def get_EAR_dependencies(
         self,
         as_objects: Optional[bool] = False,
@@ -708,6 +709,7 @@ class ElementIteration:
             out = self.workflow.get_EARs_from_IDs(out)
         return out
 
+    @TimeIt.decorator
     def get_element_iteration_dependencies(
         self, as_objects: bool = False
     ) -> List[Union[int, app.ElementIteration]]:
@@ -719,6 +721,7 @@ class ElementIteration:
             out = self.workflow.get_element_iterations_from_IDs(out)
         return out
 
+    @TimeIt.decorator
     def get_element_dependencies(
         self,
         as_objects: Optional[bool] = False,
@@ -769,6 +772,7 @@ class ElementIteration:
 
         return out
 
+    @TimeIt.decorator
     def get_dependent_EARs(
         self, as_objects: bool = False
     ) -> List[Union[int, app.ElementActionRun]]:
@@ -793,6 +797,7 @@ class ElementIteration:
 
         return deps
 
+    @TimeIt.decorator
     def get_dependent_element_iterations(
         self, as_objects: bool = False
     ) -> List[Union[int, app.ElementIteration]]:
@@ -816,6 +821,7 @@ class ElementIteration:
 
         return deps
 
+    @TimeIt.decorator
     def get_dependent_elements(
         self,
         as_objects: bool = False,
@@ -1246,6 +1252,7 @@ class Element:
         """Get tasks that depend on the most recent iteration of this element."""
         return self.latest_iteration.get_dependent_tasks(as_objects=as_objects)
 
+    @TimeIt.decorator
     def get_dependent_elements_recursively(self, task_insert_ID=None):
         """Get downstream elements that depend on this element, including recursive
         dependencies.

hpcflow/sdk/core/loop.py

@@ -6,9 +6,11 @@ from typing import Dict, List, Optional, Tuple, Union
 from hpcflow.sdk import app
 from hpcflow.sdk.core.errors import LoopTaskSubsetError
 from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
+from hpcflow.sdk.core.loop_cache import LoopCache
 from hpcflow.sdk.core.parameters import InputSourceType
 from hpcflow.sdk.core.task import WorkflowTask
-from hpcflow.sdk.core.utils import check_valid_py_identifier
+from hpcflow.sdk.core.utils import check_valid_py_identifier, nth_key, nth_value
+from hpcflow.sdk.log import TimeIt
 
 # from .parameters import Parameter
 
@@ -198,6 +200,7 @@ class WorkflowLoop:
 
         self._validate()
 
+    @TimeIt.decorator
     def _validate(self):
         # task subset must be a contiguous range of task indices:
         task_indices = self.task_indices
@@ -328,6 +331,7 @@ class WorkflowLoop:
         return self.workflow.tasks[: self.task_objects[0].index]
 
     @staticmethod
+    @TimeIt.decorator
     def _find_iterable_parameters(loop_template: app.Loop):
         all_inputs_first_idx = {}
         all_outputs_idx = {}
@@ -355,18 +359,19 @@ class WorkflowLoop:
         return iterable_params
 
     @classmethod
+    @TimeIt.decorator
     def new_empty_loop(
         cls,
         index: int,
         workflow: app.Workflow,
         template: app.Loop,
-        iterations: List[app.ElementIteration],
+        iter_loop_idx: List[Dict],
     ) -> Tuple[app.WorkflowLoop, List[Dict[str, int]]]:
         parent_loops = cls._get_parent_loops(index, workflow, template)
         parent_names = [i.name for i in parent_loops]
         num_added_iters = {}
-        for iter_i in iterations:
-            num_added_iters[tuple([iter_i.loop_idx[j] for j in parent_names])] = 1
+        for i in iter_loop_idx:
+            num_added_iters[tuple([i[j] for j in parent_names])] = 1
 
         obj = cls(
             index=index,
@@ -379,6 +384,7 @@ class WorkflowLoop:
         return obj
 
     @classmethod
+    @TimeIt.decorator
     def _get_parent_loops(
         cls,
         index: int,
@@ -399,12 +405,14 @@ class WorkflowLoop:
                 parents.append(loop_i)
         return parents
 
+    @TimeIt.decorator
     def get_parent_loops(self) -> List[app.WorkflowLoop]:
         """Get loops whose task subset is a superset of this loop's task subset. If two
         loops have identical task subsets, the first loop in the workflow loop list is
         considered the child."""
         return self._get_parent_loops(self.index, self.workflow, self.template)
 
+    @TimeIt.decorator
     def get_child_loops(self) -> List[app.WorkflowLoop]:
         """Get loops whose task subset is a subset of this loop's task subset. If two
         loops have identical task subsets, the first loop in the workflow loop list is
@@ -426,10 +434,12 @@ class WorkflowLoop:
         children = sorted(children, key=lambda x: len(next(iter(x.num_added_iterations))))
         return children
 
-    def add_iteration(self, parent_loop_indices=None):
+    @TimeIt.decorator
+    def add_iteration(self, parent_loop_indices=None, cache: Optional[LoopCache] = None):
+        if not cache:
+            cache = LoopCache.build(self.workflow)
         parent_loops = self.get_parent_loops()
         child_loops = self.get_child_loops()
-        child_loop_names = [i.name for i in child_loops]
         parent_loop_indices = parent_loop_indices or {}
         if parent_loops and not parent_loop_indices:
             parent_loop_indices = {i.name: 0 for i in parent_loops}
@@ -458,24 +468,19 @@ class WorkflowLoop:
                     if task.insert_ID in child.task_insert_IDs
                 },
             }
+            added_iter_IDs = []
             for elem_idx in range(task.num_elements):
-                # element needs to take into account changes made in this code
-                element = task.elements[elem_idx]
-                inp_statuses = task.template.get_input_statuses(element.element_set)
+
+                elem_ID = task.element_IDs[elem_idx]
+
                 new_data_idx = {}
-                existing_inners = []
-                for iter_i in element.iterations:
-                    if iter_i.loop_idx[self.name] == cur_loop_idx:
-                        existing_inner_i = {
-                            k: v
-                            for k, v in iter_i.loop_idx.items()
-                            if k in child_loop_names
-                        }
-                        if existing_inner_i:
-                            existing_inners.append(existing_inner_i)
 
                 # copy resources from zeroth iteration:
-                for key, val in element.iterations[0].get_data_idx().items():
+                zeroth_iter_ID, zi_iter_data_idx = cache.zeroth_iters[elem_ID]
+                zi_elem_ID, zi_idx = cache.iterations[zeroth_iter_ID]
+                zi_data_idx = nth_value(cache.data_idx[zi_elem_ID], zi_idx)
+
+                for key, val in zi_data_idx.items():
                     if key.startswith("resources."):
                         new_data_idx[key] = val
 
@@ -493,41 +498,47 @@ class WorkflowLoop:
                         # identify element(s) from which this iterable input should be
                         # parametrised:
                         if task.insert_ID == iter_dat["output_tasks"][-1]:
-                            src_elem = element
+                            src_elem_ID = elem_ID
                             grouped_elems = None
                         else:
-                            src_elems = element.get_dependent_elements_recursively(
-                                task_insert_ID=iter_dat["output_tasks"][-1]
-                            )
+                            src_elem_IDs_all = cache.element_dependents[elem_ID]
+                            src_elem_IDs = {
+                                k: v
+                                for k, v in src_elem_IDs_all.items()
+                                if cache.elements[k]["task_insert_ID"]
+                                == iter_dat["output_tasks"][-1]
+                            }
                             # consider groups
                             inp_group_name = inp.single_labelled_data.get("group")
                             grouped_elems = []
-                            for i in src_elems:
+                            for src_elem_j_ID, src_elem_j_dat in src_elem_IDs.items():
                                 i_in_group = any(
-                                    j.name == inp_group_name for j in i.element_set.groups
+                                    k == inp_group_name
+                                    for k in src_elem_j_dat["group_names"]
                                 )
                                 if i_in_group:
-                                    grouped_elems.append(i)
+                                    grouped_elems.append(src_elem_j_ID)
 
-                            if not grouped_elems and len(src_elems) > 1:
+                            if not grouped_elems and len(src_elem_IDs) > 1:
                                 raise NotImplementedError(
-                                    f"Multiple elements found in the iterable parameter {inp!r}'s"
-                                    f" latest output task (insert ID: "
-                                    f"{iter_dat['output_tasks'][-1]}) that can be used to "
-                                    f"parametrise the next iteration: {src_elems!r}."
+                                    f"Multiple elements found in the iterable parameter "
+                                    f"{inp!r}'s latest output task (insert ID: "
+                                    f"{iter_dat['output_tasks'][-1]}) that can be used "
+                                    f"to parametrise the next iteration: "
+                                    f"{list(src_elem_IDs.keys())!r}."
                                 )
 
-                            elif not src_elems:
+                            elif not src_elem_IDs:
                                 # TODO: maybe OK?
                                 raise NotImplementedError(
-                                    f"No elements found in the iterable parameter {inp!r}'s"
-                                    f" latest output task (insert ID: "
-                                    f"{iter_dat['output_tasks'][-1]}) that can be used to "
-                                    f"parametrise the next iteration."
+                                    f"No elements found in the iterable parameter "
+                                    f"{inp!r}'s latest output task (insert ID: "
+                                    f"{iter_dat['output_tasks'][-1]}) that can be used "
+                                    f"to parametrise the next iteration."
                                 )
 
                             else:
-                                src_elem = src_elems[0]
+                                src_elem_ID = nth_key(src_elem_IDs, 0)
 
                         child_loop_max_iters = {}
                         parent_loop_same_iters = {
@@ -553,76 +564,69 @@ class WorkflowLoop:
 
                         # identify the ElementIteration from which this input should be
                         # parametrised:
-                        source_iter = None
+                        loop_idx_key = tuple(sorted(source_iter_loop_idx.items()))
                         if grouped_elems:
-                            source_iter = []
-                            for src_elem in grouped_elems:
-                                for iter_i in src_elem.iterations:
-                                    if iter_i.loop_idx == source_iter_loop_idx:
-                                        source_iter.append(iter_i)
-                                        break
+                            src_data_idx = []
+                            for src_elem_ID in grouped_elems:
+                                src_data_idx.append(
+                                    cache.data_idx[src_elem_ID][loop_idx_key]
+                                )
                         else:
-                            for iter_i in src_elem.iterations:
-                                if iter_i.loop_idx == source_iter_loop_idx:
-                                    source_iter = iter_i
-                                    break
+                            src_data_idx = cache.data_idx[src_elem_ID][loop_idx_key]
 
-                        if not source_iter:
+                        if not src_data_idx:
                             raise RuntimeError(
                                 f"Could not find a source iteration with loop_idx: "
                                 f"{source_iter_loop_idx!r}."
                             )
 
                         if grouped_elems:
-                            inp_dat_idx = [
-                                i.get_data_idx()[f"outputs.{inp.typ}"]
-                                for i in source_iter
-                            ]
+                            inp_dat_idx = [i[f"outputs.{inp.typ}"] for i in src_data_idx]
                         else:
-                            inp_dat_idx = source_iter.get_data_idx()[f"outputs.{inp.typ}"]
+                            inp_dat_idx = src_data_idx[f"outputs.{inp.typ}"]
                         new_data_idx[f"inputs.{inp.typ}"] = inp_dat_idx
 
                     else:
                         inp_key = f"inputs.{inp.typ}"
 
-                        orig_inp_src = element.input_sources[inp_key]
+                        orig_inp_src = cache.elements[elem_ID]["input_sources"][inp_key]
                         inp_dat_idx = None
 
                         if orig_inp_src.source_type is InputSourceType.LOCAL:
                             # keep locally defined inputs from original element
-                            inp_dat_idx = element.iterations[0].get_data_idx()[inp_key]
+                            inp_dat_idx = zi_data_idx[inp_key]
 
                         elif orig_inp_src.source_type is InputSourceType.DEFAULT:
                             # keep default value from original element
-                            inp_dat_idx_iter_0 = element.iterations[0].get_data_idx()
                             try:
-                                inp_dat_idx = inp_dat_idx_iter_0[inp_key]
+                                inp_dat_idx = zi_data_idx[inp_key]
                             except KeyError:
                                 # if this input is required by a conditional action, and
                                 # that condition is not met, then this input will not
                                 # exist in the action-run data index, so use the initial
                                 # iteration data index:
-                                inp_dat_idx = element.iterations[0].data_idx[inp_key]
+                                inp_dat_idx = zi_iter_data_idx[inp_key]
 
                         elif orig_inp_src.source_type is InputSourceType.TASK:
                             if orig_inp_src.task_ref not in self.task_insert_IDs:
-                                # TODO: what about groups?
                                 # source the data_idx from the iteration with same parent
                                 # loop indices as the new iteration to add:
-                                src_iters = []
-                                for iter_i in element.iterations:
+                                # src_iters = []
+                                src_data_idx = []
+                                for li_k, di_k in cache.data_idx[elem_ID].items():
                                     skip_iter = False
+                                    li_k_dct = dict(li_k)
                                     for p_k, p_v in parent_loop_indices.items():
-                                        if iter_i.loop_idx.get(p_k) != p_v:
+                                        if li_k_dct.get(p_k) != p_v:
                                             skip_iter = True
                                             break
                                     if not skip_iter:
-                                        src_iters.append(iter_i)
+                                        src_data_idx.append(di_k)
 
                                 # could be multiple, but they should all have the same
                                 # data index for this parameter:
-                                src_iter = src_iters[0]
-                                inp_dat_idx = src_iter.get_data_idx()[inp_key]
+                                src_data_idx = src_data_idx[0]
+                                inp_dat_idx = src_data_idx[inp_key]
                             else:
                                 is_group = False
                                 if (
@@ -645,19 +649,24 @@ class WorkflowLoop:
                                         # find which element in that task `element`
                                         # depends on:
                                         task_i = self.workflow.tasks.get(insert_ID=tiID)
-                                        elem_i = task_i.elements[e_idx]
-                                        src_elems_i = (
-                                            elem_i.get_dependent_elements_recursively(
-                                                task_insert_ID=task.insert_ID
-                                            )
-                                        )
-                                        # filter src_elems_i for matching element IDs:
-                                        src_elems_i = [
-                                            i for i in src_elems_i if i.id_ == element.id_
+                                        elem_i_ID = task_i.element_IDs[e_idx]
+                                        src_elem_IDs_all = cache.element_dependents[
+                                            elem_i_ID
+                                        ]
+                                        src_elem_IDs_i = {
+                                            k: v
+                                            for k, v in src_elem_IDs_all.items()
+                                            if cache.elements[k]["task_insert_ID"]
+                                            == task.insert_ID
+                                        }
+
+                                        # filter src_elem_IDs_i for matching element IDs:
+                                        src_elem_IDs_i = [
+                                            i for i in src_elem_IDs_i if i == elem_ID
                                         ]
                                         if (
-                                            len(src_elems_i) == 1
-                                            and src_elems_i[0].id_ == element.id_
+                                            len(src_elem_IDs_i) == 1
+                                            and src_elem_IDs_i[0] == elem_ID
                                         ):
                                             new_sources.append((tiID, e_idx))
 
@@ -680,10 +689,11 @@ class WorkflowLoop:
                         new_data_idx[inp_key] = inp_dat_idx
 
                 # add any locally defined sub-parameters:
+                inp_statuses = cache.elements[elem_ID]["input_statuses"]
                 inp_status_inps = set([f"inputs.{i}" for i in inp_statuses])
                 sub_params = inp_status_inps - set(new_data_idx.keys())
                 for sub_param_i in sub_params:
-                    sub_param_data_idx_iter_0 = element.iterations[0].get_data_idx()
+                    sub_param_data_idx_iter_0 = zi_data_idx
                     try:
                         sub_param_data_idx = sub_param_data_idx_iter_0[sub_param_i]
                     except KeyError:
@@ -691,7 +701,7 @@ class WorkflowLoop:
                         # and that condition is not met, then this input will not exist in
                         # the action-run data index, so use the initial iteration data
                         # index:
-                        sub_param_data_idx = element.iterations[0].data_idx[sub_param_i]
+                        sub_param_data_idx = zi_data_idx[sub_param_i]
 
                     new_data_idx[sub_param_i] = sub_param_data_idx
 
@@ -703,16 +713,26 @@ class WorkflowLoop:
                 schema_params = set(
                     i for i in new_data_idx.keys() if len(i.split(".")) == 2
                 )
-                all_new_data_idx[(task.insert_ID, element.index)] = new_data_idx
+                all_new_data_idx[(task.insert_ID, elem_idx)] = new_data_idx
 
                 iter_ID_i = self.workflow._store.add_element_iteration(
-                    element_ID=element.id_,
+                    element_ID=elem_ID,
                     data_idx=new_data_idx,
                     schema_parameters=list(schema_params),
                     loop_idx=new_loop_idx,
                 )
+                if cache:
+                    cache.add_iteration(
+                        iter_ID=iter_ID_i,
+                        task_insert_ID=task.insert_ID,
+                        element_ID=elem_ID,
+                        loop_idx=new_loop_idx,
+                        data_idx=new_data_idx,
+                    )
 
-                task.initialise_EARs()
+                added_iter_IDs.append(iter_ID_i)
+
+            task.initialise_EARs(iter_IDs=added_iter_IDs)
 
         added_iters_key = tuple(parent_loop_indices[k] for k in self.parents)
         self._increment_pending_added_iters(added_iters_key)
@@ -731,7 +751,8 @@ class WorkflowLoop:
                             **par_idx,
                             **parent_loop_indices,
                             self.name: cur_loop_idx + 1,
-                        }
+                        },
+                        cache=cache,
                     )
 
     def test_termination(self, element_iter):

hpcflow/sdk/core/loop_cache.py

@@ -0,0 +1,140 @@
+from dataclasses import dataclass
+from collections import defaultdict
+from typing import Dict, List, Optional, Tuple
+
+from hpcflow.sdk import app
+from hpcflow.sdk.core.utils import nth_key
+from hpcflow.sdk.log import TimeIt
+from hpcflow.sdk.core.cache import DependencyCache
+
+
+@dataclass
+class LoopCache:
+    """Class to store a cache for use in `Workflow.add_empty_loop` and
+    `WorkflowLoop.add_iterations`.
+
+    Attributes
+    ----------
+    element_dependents
+        Keys are element IDs, values are dicts whose keys are element IDs that depend on
+        the key element ID (via `Element.get_dependent_elements_recursively`), and whose
+        values are dicts with keys: `group_names`, which is a tuple of the string group
+        names associated with the dependent element's element set.
+    elements
+        Keys are element IDs, values are dicts with keys: `input_statuses`,
+        `input_sources`, and `task_insert_ID`.
+    zeroth_iters
+        Keys are element IDs, values are data associated with the zeroth iteration of that
+        element, namely a tuple of iteration ID and `ElementIteration.data_idx`.
+    data_idx
+        Keys are element IDs, values are data associated with all iterations of that
+        element, namely a dict whose keys are the iteration loop index as a tuple, and
+        whose values are data indices via `ElementIteration.get_data_idx()`.
+    iterations
+        Keys are iteration IDs, values are tuples of element ID and iteration index within
+        that element.
+    task_iterations
+        Keys are task insert IDs, values are list of all iteration IDs associated with
+        that task.
+
+    """
+
+    element_dependents: Dict[int, Dict]
+    elements: Dict[int, Dict]
+    zeroth_iters: Dict[int, Tuple]
+    data_idx: Dict[int, Dict]
+    iterations: Dict[int, Tuple]
+    task_iterations: Dict[int, List[int]]
+
+    @TimeIt.decorator
+    def get_iter_IDs(self, loop: "app.Loop") -> List[int]:
+        """Retrieve a list of iteration IDs belonging to a given loop."""
+        return [j for i in loop.task_insert_IDs for j in self.task_iterations[i]]
+
+    @TimeIt.decorator
+    def get_iter_loop_indices(self, iter_IDs: List[int]) -> List[Dict[str, int]]:
+        iter_loop_idx = []
+        for i in iter_IDs:
+            elem_id, idx = self.iterations[i]
+            iter_loop_idx.append(dict(nth_key(self.data_idx[elem_id], idx)))
+        return iter_loop_idx
+
+    @TimeIt.decorator
+    def update_loop_indices(self, new_loop_name: str, iter_IDs: List[int]):
+        elem_ids = {v[0] for k, v in self.iterations.items() if k in iter_IDs}
+        for i in elem_ids:
+            new_item = {}
+            for k, v in self.data_idx[i].items():
+                new_k = dict(k)
+                new_k.update({new_loop_name: 0})
+                new_item[tuple(sorted(new_k.items()))] = v
+            self.data_idx[i] = new_item
+
+    @TimeIt.decorator
+    def add_iteration(self, iter_ID, task_insert_ID, element_ID, loop_idx, data_idx):
+        """Update the cache to include a newly added iteration."""
+        self.task_iterations[task_insert_ID].append(iter_ID)
+        new_iter_idx = len(self.data_idx[element_ID])
+        self.data_idx[element_ID][tuple(sorted(loop_idx.items()))] = data_idx
+        self.iterations[iter_ID] = (element_ID, new_iter_idx)
+
+    @classmethod
+    @TimeIt.decorator
+    def build(cls, workflow: "app.Workflow", loops: Optional[List["app.Loop"]] = None):
+        """Build a cache of data for use in adding loops and iterations."""
+
+        deps_cache = DependencyCache.build(workflow)
+
+        loops = list(workflow.template.loops) + (loops or [])
+        task_iIDs = set(j for i in loops for j in i.task_insert_IDs)
+        tasks = [workflow.tasks.get(insert_ID=i) for i in sorted(task_iIDs)]
+        elem_deps = {}
+
+        # keys: element IDs, values: dict with keys: tuple(loop_idx), values: data index
+        data_idx_cache = {}
+
+        # keys: iteration IDs, values: tuple of (element ID, integer index into values
+        # dict in `data_idx_cache` [accessed via `.keys()[index]`])
+        iters = {}
+
+        # keys: element IDs, values: dict with keys: "input_statues", "input_sources",
+        # "task_insert_ID":
+        elements = {}
+
+        zeroth_iters = {}
+        task_iterations = defaultdict(list)
+        for task in tasks:
+            for elem_idx in task.element_IDs:
+                element = deps_cache.elements[elem_idx]
+                inp_statuses = task.template.get_input_statuses(element.element_set)
+                elements[element.id_] = {
+                    "input_statuses": inp_statuses,
+                    "input_sources": element.input_sources,
+                    "task_insert_ID": task.insert_ID,
+                }
+                elem_deps[element.id_] = {
+                    i: {
+                        "group_names": tuple(
+                            j.name for j in deps_cache.elements[i].element_set.groups
+                        ),
+                    }
+                    for i in deps_cache.elem_elem_dependents_rec[element.id_]
+                }
+                elem_iters = {}
+                for idx, iter_i in enumerate(element.iterations):
+                    if idx == 0:
+                        zeroth_iters[element.id_] = (iter_i.id_, iter_i.data_idx)
+                    loop_idx_key = tuple(sorted(iter_i.loop_idx.items()))
+                    elem_iters[loop_idx_key] = iter_i.get_data_idx()
+                    task_iterations[task.insert_ID].append(iter_i.id_)
+                    iters[iter_i.id_] = (element.id_, idx)
+                data_idx_cache[element.id_] = elem_iters
+
+        return cls(
+            element_dependents=elem_deps,
+            elements=elements,
+            zeroth_iters=zeroth_iters,
+            data_idx=data_idx_cache,
+            iterations=iters,
+            task_iterations=dict(task_iterations),
+        )

hpcflow/sdk/core/task.py

@@ -2062,29 +2062,36 @@ class WorkflowTask:
         return element_dat_idx
 
     @TimeIt.decorator
-    def initialise_EARs(self) -> List[int]:
+    def initialise_EARs(self, iter_IDs: Optional[List[int]] = None) -> List[int]:
         """Try to initialise any uninitialised EARs of this task."""
+        if iter_IDs:
+            iters = self.workflow.get_element_iterations_from_IDs(iter_IDs)
+        else:
+            iters = []
+            for element in self.elements:
+                # We don't yet cache Element objects, so `element`, and also it's
+                # `ElementIterations, are transient. So there is no reason to update these
+                # objects in memory to account for the new EARs. Subsequent calls to
+                # `WorkflowTask.elements` will retrieve correct element data from the
+                # store. This might need changing once/if we start caching Element
+                # objects.
+                iters.extend(element.iterations)
+
         initialised = []
-        for element in self.elements[:]:
-            # We don't yet cache Element objects, so `element`, and also it's
-            # `ElementIterations, are transient. So there is no reason to update these
-            # objects in memory to account for the new EARs. Subsequent calls to
-            # `WorkflowTask.elements` will retrieve correct element data from the store.
-            # This might need changing once/if we start caching Element objects.
-            for iter_i in element.iterations:
-                if not iter_i.EARs_initialised:
-                    try:
-                        self._initialise_element_iter_EARs(iter_i)
-                        initialised.append(iter_i.id_)
-                    except UnsetParameterDataError:
-                        # raised by `Action.test_rules`; cannot yet initialise EARs
-                        self.app.logger.debug(
-                            f"UnsetParameterDataError raised: cannot yet initialise runs."
-                        )
-                        pass
-                    else:
-                        iter_i._EARs_initialised = True
-                        self.workflow.set_EARs_initialised(iter_i.id_)
+        for iter_i in iters:
+            if not iter_i.EARs_initialised:
+                try:
+                    self._initialise_element_iter_EARs(iter_i)
+                    initialised.append(iter_i.id_)
+                except UnsetParameterDataError:
+                    # raised by `Action.test_rules`; cannot yet initialise EARs
+                    self.app.logger.debug(
+                        f"UnsetParameterDataError raised: cannot yet initialise runs."
+                    )
+                    pass
+                else:
+                    iter_i._EARs_initialised = True
+                    self.workflow.set_EARs_initialised(iter_i.id_)
         return initialised
 
     @TimeIt.decorator
@@ -2097,7 +2104,6 @@ class WorkflowTask:
         param_src_updates = {}
 
         count = 0
-        # TODO: generator is an IO op here, can be pre-calculated/cached?
         for act_idx, action in self.template.all_schema_actions():
             log_common = (
                 f"for action {act_idx} of element iteration {element_iter.index} of "
@@ -2151,8 +2157,7 @@ class WorkflowTask:
                 metadata={},
             )
 
-        for pid, src in param_src_updates.items():
-            self.workflow._store.update_param_source(pid, src)
+        self.workflow._store.update_param_source(param_src_updates)
 
     @TimeIt.decorator
     def _add_element_set(self, element_set):

hpcflow/sdk/core/utils.py

@@ -3,7 +3,7 @@ import enum
 from functools import wraps
 import contextlib
 import hashlib
-from itertools import accumulate
+from itertools import accumulate, islice
 import json
 import keyword
 import os
@@ -871,3 +871,13 @@ def dict_values_process_flat(d, callable):
         out[k] = proc_idx_k
 
     return out
+
+
+def nth_key(dct, n):
+    it = iter(dct)
+    next(islice(it, n, n), None)
+    return next(it)
+
+
+def nth_value(dct, n):
+    return dct[nth_key(dct, n)]

hpcflow/sdk/core/workflow.py

@@ -25,6 +25,7 @@ from hpcflow.sdk.core import (
     ABORT_EXIT_CODE,
 )
 from hpcflow.sdk.core.actions import EARStatus
+from hpcflow.sdk.core.loop_cache import LoopCache
 from hpcflow.sdk.log import TimeIt
 from hpcflow.sdk.persistence import store_cls_from_str, DEFAULT_STORE_FORMAT
 from hpcflow.sdk.persistence.base import TEMPLATE_COMP_TYPES, AnySEAR
@@ -41,6 +42,7 @@ from hpcflow.sdk.submission.schedulers.direct import DirectScheduler
 from hpcflow.sdk.typing import PathLike
 from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
 from .utils import (
+    nth_key,
     read_JSON_file,
     read_JSON_string,
     read_YAML_str,
@@ -625,19 +627,28 @@ class Workflow:
             )
             with wk._store.cached_load():
                 with wk.batch_update(is_workflow_creation=True):
-                    for idx, task in enumerate(template.tasks):
+                    with wk._store.cache_ctx():
+                        for idx, task in enumerate(template.tasks):
+                            if status:
+                                status.update(
+                                    f"Adding task {idx + 1}/{len(template.tasks)} "
+                                    f"({task.name!r})..."
+                                )
+                            wk._add_task(task)
                         if status:
                             status.update(
-                                f"Adding task {idx + 1}/{len(template.tasks)} "
-                                f"({task.name!r})..."
+                                f"Preparing to add {len(template.loops)} loops..."
                             )
-                        wk._add_task(task)
-                    for idx, loop in enumerate(template.loops):
-                        if status:
-                            status.update(
-                                f"Adding loop {idx + 1}/" f"{len(template.loops)}..."
-                            )
-                        wk._add_loop(loop)
+                        if template.loops:
+                            # TODO: if loop with non-initialisable actions, will fail
+                            cache = LoopCache.build(workflow=wk, loops=template.loops)
+                            for idx, loop in enumerate(template.loops):
+                                if status:
+                                    status.update(
+                                        f"Adding loop {idx + 1}/"
+                                        f"{len(template.loops)} ({loop.name!r})"
+                                    )
+                                wk._add_loop(loop, cache=cache, status=status)
         except Exception:
             if status:
                 status.stop()
@@ -1101,7 +1112,7 @@ class Workflow:
 
     @TimeIt.decorator
     def _add_empty_loop(
-        self, loop: app.Loop
+        self, loop: app.Loop, cache: LoopCache
     ) -> Tuple[app.WorkflowLoop, List[app.ElementIteration]]:
         """Add a new loop (zeroth iterations only) to the workflow."""
 
@@ -1114,15 +1125,15 @@ class Workflow:
         self.template._add_empty_loop(loop_c)
 
         # all these element iterations will be initialised for the new loop:
-        iters = self.get_element_iterations_of_tasks(loop_c.task_insert_IDs)
-        iter_IDs = [i.id_ for i in iters]
+        iter_IDs = cache.get_iter_IDs(loop_c)
+        iter_loop_idx = cache.get_iter_loop_indices(iter_IDs)
 
         # create and insert a new WorkflowLoop:
         new_loop = self.app.WorkflowLoop.new_empty_loop(
             index=new_index,
             workflow=self,
             template=loop_c,
-            iterations=iters,
+            iter_loop_idx=iter_loop_idx,
         )
         self.loops.add_object(new_loop)
         wk_loop = self.loops[new_index]
@@ -1144,15 +1155,28 @@ class Workflow:
 
         self._pending["loops"].append(new_index)
 
+        # update cache loop indices:
+        cache.update_loop_indices(new_loop_name=loop_c.name, iter_IDs=iter_IDs)
+
         return wk_loop
 
     @TimeIt.decorator
-    def _add_loop(self, loop: app.Loop) -> None:
-        new_wk_loop = self._add_empty_loop(loop)
+    def _add_loop(
+        self, loop: app.Loop, cache: Optional[Dict] = None, status: Optional[Any] = None
+    ) -> None:
+        if not cache:
+            cache = LoopCache.build(workflow=self, loops=[loop])
+        new_wk_loop = self._add_empty_loop(loop, cache)
         if loop.num_iterations is not None:
             # fixed number of iterations, so add remaining N > 0 iterations:
-            for _ in range(loop.num_iterations - 1):
-                new_wk_loop.add_iteration()
+            if status:
+                status_prev = status.status
+            for iter_idx in range(loop.num_iterations - 1):
+                if status:
+                    status.update(
+                        f"{status_prev}: iteration {iter_idx + 2}/{loop.num_iterations}."
+                    )
+                new_wk_loop.add_iteration(cache=cache)
 
     def add_loop(self, loop: app.Loop) -> None:
         """Add a loop to a subset of workflow tasks."""
@@ -1326,6 +1350,7 @@ class Workflow:
                     iters.append(iter_i)
         return iters
 
+    @TimeIt.decorator
     def get_elements_from_IDs(self, id_lst: Iterable[int]) -> List[app.Element]:
         """Return element objects from a list of IDs."""
 
@@ -1334,6 +1359,7 @@ class Workflow:
         task_IDs = [i.task_ID for i in store_elems]
         store_tasks = self._store.get_tasks_by_IDs(task_IDs)
 
+        element_idx_by_task = defaultdict(set)
         index_paths = []
         for el, tk in zip(store_elems, store_tasks):
             elem_idx = tk.element_IDs.index(el.id_)
@@ -1343,15 +1369,23 @@ class Workflow:
                     "task_idx": tk.index,
                 }
             )
+            element_idx_by_task[tk.index].add(elem_idx)
+
+        elements_by_task = {}
+        for task_idx, elem_idx in element_idx_by_task.items():
+            task = self.tasks[task_idx]
+            elements_by_task[task_idx] = dict(
+                zip(elem_idx, task.elements[list(elem_idx)])
+            )
 
         objs = []
         for idx_dat in index_paths:
-            task = self.tasks[idx_dat["task_idx"]]
-            elem = task.elements[idx_dat["elem_idx"]]
+            elem = elements_by_task[idx_dat["task_idx"]][idx_dat["elem_idx"]]
             objs.append(elem)
 
         return objs
 
+    @TimeIt.decorator
     def get_element_iterations_from_IDs(
         self, id_lst: Iterable[int]
     ) -> List[app.ElementIteration]:
@@ -1365,6 +1399,8 @@ class Workflow:
         task_IDs = [i.task_ID for i in store_elems]
         store_tasks = self._store.get_tasks_by_IDs(task_IDs)
 
+        element_idx_by_task = defaultdict(set)
+
         index_paths = []
         for it, el, tk in zip(store_iters, store_elems, store_tasks):
             iter_idx = el.iteration_IDs.index(it.id_)
@@ -1376,11 +1412,18 @@ class Workflow:
                     "task_idx": tk.index,
                 }
             )
+            element_idx_by_task[tk.index].add(elem_idx)
+
+        elements_by_task = {}
+        for task_idx, elem_idx in element_idx_by_task.items():
+            task = self.tasks[task_idx]
+            elements_by_task[task_idx] = dict(
+                zip(elem_idx, task.elements[list(elem_idx)])
+            )
 
         objs = []
         for idx_dat in index_paths:
-            task = self.tasks[idx_dat["task_idx"]]
-            elem = task.elements[idx_dat["elem_idx"]]
+            elem = elements_by_task[idx_dat["task_idx"]][idx_dat["elem_idx"]]
             iter_ = elem.iterations[idx_dat["iter_idx"]]
             objs.append(iter_)
 

hpcflow/sdk/persistence/base.py

@@ -716,6 +716,11 @@ class PersistentStore(ABC):
         """Cache for number of persistent tasks."""
         return self._cache["num_tasks"]
 
+    @property
+    def num_EARs_cache(self):
+        """Cache for total number of persistent EARs."""
+        return self._cache["num_EARs"]
+
     @property
     def param_sources_cache(self):
         """Cache for persistent parameter sources."""
@@ -730,6 +735,10 @@ class PersistentStore(ABC):
     def num_tasks_cache(self, value):
         self._cache["num_tasks"] = value
 
+    @num_EARs_cache.setter
+    def num_EARs_cache(self, value):
+        self._cache["num_EARs"] = value
+
     def _reset_cache(self):
         self._cache = {
             "tasks": {},
@@ -739,6 +748,7 @@ class PersistentStore(ABC):
             "param_sources": {},
             "num_tasks": None,
             "parameters": {},
+            "num_EARs": None,
         }
 
     @contextlib.contextmanager
@@ -873,6 +883,7 @@ class PersistentStore(ABC):
         """Get the total number of persistent and pending element iterations."""
         return self._get_num_persistent_elem_iters() + len(self._pending.add_elem_iters)
 
+    @TimeIt.decorator
     def _get_num_total_EARs(self):
         """Get the total number of persistent and pending EARs."""
         return self._get_num_persistent_EARs() + len(self._pending.add_EARs)
@@ -1296,9 +1307,11 @@ class PersistentStore(ABC):
             self.save()
 
     @TimeIt.decorator
-    def update_param_source(self, param_id: int, source: Dict, save: bool = True) -> None:
-        self.logger.debug(f"Updating parameter ID {param_id!r} source to {source!r}.")
-        self._pending.update_param_sources[param_id] = source
+    def update_param_source(
+        self, param_sources: Dict[int, Dict], save: bool = True
+    ) -> None:
+        self.logger.debug(f"Updating parameter sources with {param_sources!r}.")
+        self._pending.update_param_sources.update(param_sources)
         if save:
             self.save()
 

hpcflow/sdk/persistence/json.py

@@ -303,12 +303,13 @@ class JSONPersistentStore(PersistentStore):
 
     def _get_num_persistent_tasks(self) -> int:
         """Get the number of persistent tasks."""
-        if self.num_tasks_cache is not None:
+        if self.use_cache and self.num_tasks_cache is not None:
             num = self.num_tasks_cache
         else:
             with self.using_resource("metadata", action="read") as md:
                 num = len(md["tasks"])
-                self.num_tasks_cache = num
+        if self.use_cache and self.num_tasks_cache is None:
+            self.num_tasks_cache = num
         return num
 
     def _get_num_persistent_loops(self) -> int:
@@ -333,8 +334,14 @@ class JSONPersistentStore(PersistentStore):
 
     def _get_num_persistent_EARs(self) -> int:
         """Get the number of persistent EARs."""
-        with self.using_resource("metadata", action="read") as md:
-            return len(md["runs"])
+        if self.use_cache and self.num_EARs_cache is not None:
+            num = self.num_EARs_cache
+        else:
+            with self.using_resource("metadata", action="read") as md:
+                num = len(md["runs"])
+        if self.use_cache and self.num_EARs_cache is None:
+            self.num_EARs_cache = num
+        return num
 
     def _get_num_persistent_parameters(self):
         with self.using_resource("parameters", "read") as params:

hpcflow/sdk/persistence/pending.py

@@ -275,6 +275,7 @@ class PendingChanges:
             EAR_ids = list(self.add_EARs.keys())
             self.logger.debug(f"commit: adding pending EARs with IDs: {EAR_ids!r}")
             self.store._append_EARs(EARs)
+            self.store.num_EARs_cache = None  # invalidate cache
             # pending start/end times/snapshots, submission indices, and skips that belong
             # to pending EARs are now committed (accounted for in `get_EARs` above):
             self.set_EAR_submission_indices = {
@@ -408,6 +409,7 @@ class PendingChanges:
     @TimeIt.decorator
     def commit_loop_indices(self) -> None:
         """Make pending update to element iteration loop indices persistent."""
+        # TODO: batch up
         for iter_ID, loop_idx in self.update_loop_indices.items():
             self.logger.debug(
                 f"commit: updating loop indices of iteration ID {iter_ID!r} with "

hpcflow/sdk/persistence/zarr.py

@@ -774,9 +774,16 @@ class ZarrPersistentStore(PersistentStore):
         """Get the number of persistent element iterations."""
         return len(self._get_iters_arr())
 
+    @TimeIt.decorator
     def _get_num_persistent_EARs(self) -> int:
         """Get the number of persistent EARs."""
-        return len(self._get_EARs_arr())
+        if self.use_cache and self.num_EARs_cache is not None:
+            num = self.num_EARs_cache
+        else:
+            num = len(self._get_EARs_arr())
+        if self.use_cache and self.num_EARs_cache is None:
+            self.num_EARs_cache = num
+        return num
 
     def _get_num_persistent_parameters(self):
         return len(self._get_parameter_base_array())

hpcflow/tests/unit/test_utils.py

@@ -13,6 +13,8 @@ from hpcflow.sdk.core.utils import (
     get_nested_indices,
     is_fsspec_url,
     linspace_rect,
+    nth_key,
+    nth_value,
     process_string_nodes,
     replace_items,
     check_valid_py_identifier,
@@ -556,3 +558,22 @@ def test_dict_values_process_flat_single_item_lists():
         "b": [4],
         "c": [5],
     }
+
+
+def test_nth_key():
+    dct = {"a": 1, "b": 2}
+    assert [nth_key(dct, i) for i in range(len(dct))] == ["a", "b"]
+
+
+def test_nth_value():
+    dct = {"a": 1, "b": 2}
+    assert [nth_value(dct, i) for i in range(len(dct))] == [1, 2]
+
+
+def test_nth_key_raises():
+    dct = {"a": 1, "b": 2}
+    with pytest.raises(Exception):
+        nth_key(dct, 2)
+
+    with pytest.raises(Exception):
+        nth_key(dct, -1)

{hpcflow_new2-0.2.0a176.dist-info → hpcflow_new2-0.2.0a177.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a176
+Version: 0.2.0a177
 Summary: Computational workflow management
 License: MIT
 Author: aplowman

{hpcflow_new2-0.2.0a176.dist-info → hpcflow_new2-0.2.0a177.dist-info}/RECORD

@@ -1,7 +1,7 @@
 hpcflow/__init__.py,sha256=WIETuRHeOp2SqUqHUzpjQ-lk9acbYv-6aWOhZPRdlhs,64
 hpcflow/__pyinstaller/__init__.py,sha256=YOzBlPSck6slucv6lJM9K80JtsJWxXRL00cv6tRj3oc,98
 hpcflow/__pyinstaller/hook-hpcflow.py,sha256=SeMopsPkhCyd9gqIrzwFNRj3ZlkUlUYl-74QYz61mo4,1089
-hpcflow/_version.py,sha256=SYxreQlS-GrGdoMoqVR3gpikj6-V4k7yVrXGBKl-Xxg,26
+hpcflow/_version.py,sha256=c6sVLAvL-ZtlEF_CBbI6lCJoA-qh3KPMN5Smq6FFaPk,26
 hpcflow/app.py,sha256=d-kgfnZNlqlCi2H8bK26714brD_u3ibN3FaEZgjF9aA,1332
 hpcflow/cli.py,sha256=G2J3D9v6MnMWOWMMWK6UEKLn_6wnV9lT_qygEBBxg-I,66
 hpcflow/data/demo_data_manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -46,24 +46,26 @@ hpcflow/sdk/config/config_file.py,sha256=JlMcprj0aujFVk8552ahP2f8EXB0tglMaHwzbcG
 hpcflow/sdk/config/errors.py,sha256=2D7HJ1dbyeoD3xk4MuaGSsbJsUyQzyw8kaThEBZfP2I,6876
 hpcflow/sdk/core/__init__.py,sha256=GcIklEsXy3M5PWpmxyhd2KoI0u6HjXRIjD_aR1bgRjo,215
 hpcflow/sdk/core/actions.py,sha256=66CHgwYAB0oCR6oB5bNbBdUGRGTU3juS1XcMNjj3vP0,77068
+hpcflow/sdk/core/cache.py,sha256=MDzqsCg8uMjxEdQ-8ta-uG042yiPrzQoVKMeE6jYW8k,5127
 hpcflow/sdk/core/command_files.py,sha256=GEFlgZv7g9lkFoNgwyDtmlI_90e2TWliCJuJimnJZts,18685
 hpcflow/sdk/core/commands.py,sha256=5SKxSBuYz8sSvfpp9p5utBwMoQV6Pd2KlGBCpXAHDxE,12741
-hpcflow/sdk/core/element.py,sha256=hTAR2kxfGSRf4vFgWwrnyuP5z5RnKYOd2X6c6Xd70zo,47048
+hpcflow/sdk/core/element.py,sha256=kWEbGWzrXCwhQ1Ie1RFm1v5_q3MQkCDPEIp01nHIf1Q,47202
 hpcflow/sdk/core/environment.py,sha256=DGUz1NvliKh6opP0IueGHD69rn_8wFLhDsq6kAmEgM4,4849
 hpcflow/sdk/core/errors.py,sha256=ku4wwsrmxBpJBFflUeZD6vrmAqgC7H02VdlRG4aAGqQ,9292
 hpcflow/sdk/core/json_like.py,sha256=LRZsUd1tn8zXC8fESeiXs7Eko-VdnB8zcXiqixKVcZM,18874
-hpcflow/sdk/core/loop.py,sha256=SYlkmjvxifBfykhjgXCbxnCvn37BfXGMRIbyOholrEE,30936
+hpcflow/sdk/core/loop.py,sha256=vj3b0jRCJxkKdhURYTgULoDJ6U3LzAYZMXBzqcCMHr8,31506
+hpcflow/sdk/core/loop_cache.py,sha256=BBmJn_pS11gaiHS8qvujBpzWLzPsfs8N6iYIBkZtIwI,5881
 hpcflow/sdk/core/object_list.py,sha256=HASx7AMniX82bTlROIgIvrjE_DupmwDgxfkfROmI3GA,20168
 hpcflow/sdk/core/parallel.py,sha256=LI-g-qOuOR1oaEUWVT0qW0hmiP9hsJyUP8_IfSTKYYo,95
 hpcflow/sdk/core/parameters.py,sha256=0h1M-fXqOVgruyM0Au7Fo38cUbHgDNEPd1Alb1FULxE,65588
 hpcflow/sdk/core/rule.py,sha256=3jVsSZCBv4Odxy8QbSbKo9ZcRuU-5DRJoNK8adXCEpI,4567
 hpcflow/sdk/core/run_dir_files.py,sha256=_k-hA7dlry9GZw5ZXcntFcPGxg07p03hnHSM5S-2G2Y,2197
-hpcflow/sdk/core/task.py,sha256=-Ugs13wl76FxBv9q0Ik2dWGSDPZQ4KR4qvwmrfsAEgE,122247
+hpcflow/sdk/core/task.py,sha256=TTAn9aeJOLyso7t11wt87wxPDVi037vwpFgF9rCfZwQ,122319
 hpcflow/sdk/core/task_schema.py,sha256=TipXzC2guu9zilv0En-rHt6lUCTSIj5faI4lVWQdUbA,32346
 hpcflow/sdk/core/test_utils.py,sha256=IhCLvRzDuG4hVNGeGulGKfZEgg7Ow-vgiEqewzMiaZ4,9762
-hpcflow/sdk/core/utils.py,sha256=pReOwnmuxJqexPUdaA8UMjJ4o8ucllBVVssWjb_LNQc,25651
+hpcflow/sdk/core/utils.py,sha256=cpwfoHgbHanZQXmVZRN3VRW8X-zZxb1I6T0v2tWgBK0,25811
 hpcflow/sdk/core/validation.py,sha256=KBKiy5DdfGiGmMaB0HdKTY0V972u5dJzvkYkX0_KtCo,518
-hpcflow/sdk/core/workflow.py,sha256=oi9QdBgAMSr62c6HQx2OXw3ljRmxhEb_3YNXGCwIbBE,111164
+hpcflow/sdk/core/workflow.py,sha256=ziKn1cA4s_eHKPMzyKfHF4bVNF7bfho4dko5qtyZKjU,113111
 hpcflow/sdk/core/zarr_io.py,sha256=V_Zm6uSiuaCbXyHFJUO74K1pAr4Zqrj3aLCBjohCwvs,5724
 hpcflow/sdk/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hpcflow/sdk/data/config_file_schema.yaml,sha256=7i3z_m3GBRtLyB4c7qPngnlQWqcIq1CyCcOysDyq4es,791
@@ -81,12 +83,12 @@ hpcflow/sdk/helper/helper.py,sha256=MkjYKHox1F4XOpy-20sCCDUTWUbQY84QpWZkcpSq9n8,
 hpcflow/sdk/helper/watcher.py,sha256=hLqgwXtZw-6ihNUUcWYnZw8TCyD_AdhYE7abOrO2r_0,4003
 hpcflow/sdk/log.py,sha256=_DA5nNS8BoSIFB3d9nrIjbxNDxFflEaL3Ubkq8UYQK8,5735
 hpcflow/sdk/persistence/__init__.py,sha256=IzWycfiO6rDn_7Kocw4Df5ETe9BSoaqqxG7Yp4FW_ls,900
-hpcflow/sdk/persistence/base.py,sha256=cmfvgUQAp6BVcOL04IemzmtPvzSJfVd2nQFvgp2VLyU,61718
-hpcflow/sdk/persistence/json.py,sha256=XD4km426WZ6F6YCrnw87ntS_Qfg3qa9CAyK2GQmJNFU,21700
-hpcflow/sdk/persistence/pending.py,sha256=ZaiY_I6c0CpXGPOkHfs595GtDvx8daP4nqVZGSJ6tS4,25566
+hpcflow/sdk/persistence/base.py,sha256=cvk2Uqd671ZFe6JEP_UrZ7W0q2mZjimyA8DFvss8hdo,62030
+hpcflow/sdk/persistence/json.py,sha256=55F4Txa50I9HzfETPqwo6gAOBRaoewMHGR3V4-2Fifc,22013
+hpcflow/sdk/persistence/pending.py,sha256=XktGkRpJmlyjceOiDY0GEL2xRl6k-gbjt057LmEj5oc,25656
 hpcflow/sdk/persistence/store_resource.py,sha256=oEyocRqa8Uym-57UFosrwate-Xw9O7i2FM82TxHc4m0,4307
 hpcflow/sdk/persistence/utils.py,sha256=yQT6gS-Ipj2N6grtlV5d0czxxKE0CaeqAkXA1247XGo,1522
-hpcflow/sdk/persistence/zarr.py,sha256=cyhLVaHUVLuFg-sokfD8L2YOMvfCCk1KCawRD9_qg30,45452
+hpcflow/sdk/persistence/zarr.py,sha256=otZbR17O3ZNIot7uVAP1UdWs_L0PHS682LuWk3z1_1g,45708
 hpcflow/sdk/runtime.py,sha256=_in5ojiy9R8fD1ZNbdE6PDmZx6kSaiG9WPB6kVBFE7k,9217
 hpcflow/sdk/submission/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hpcflow/sdk/submission/jobscript.py,sha256=Z9NUzkIcmoFw-XAtG8FdLpO2LtMt3czk1v1BnbM1eZw,44678
@@ -142,14 +144,14 @@ hpcflow/tests/unit/test_slurm.py,sha256=ewfNuXXUEEelAxcd7MBbAQ-RCvU8xBenHTAyfXYF
 hpcflow/tests/unit/test_submission.py,sha256=kQ3ksjGlfp47AYuwTA27RDX2XxRU3YxKlKC1ACTbXw8,16682
 hpcflow/tests/unit/test_task.py,sha256=QJuEpJ0y0nBesprgoau5R2kFZBCW-ygNmYatLT_M5-o,80227
 hpcflow/tests/unit/test_task_schema.py,sha256=j5HHxoqq4Mb223jKcusgX-C6-TsmKG0PLjYQ4M01ZHo,4531
-hpcflow/tests/unit/test_utils.py,sha256=JMhSRZFqmI9ZhREJet9en_y3aRVlQlWE7OKpkdt8SVI,14172
+hpcflow/tests/unit/test_utils.py,sha256=RH3UZ99g1pKKJme1rNgzT3j_txWLT9_OWE1lWx67W5M,14610
 hpcflow/tests/unit/test_value_sequence.py,sha256=yJh5YRxN-VYMbCWiUaLH4T_Ue5F2IfVS3e11zx6HlS0,15740
 hpcflow/tests/unit/test_workflow.py,sha256=Eyr9BhnsFisAPotEAeYrAvxXT1d2i6oshEh1_OxgnSc,22732
 hpcflow/tests/unit/test_workflow_template.py,sha256=fF7LNveMwCledgncNCRfD9Nd9dL9tSPtlAAOKV3ovAU,5396
 hpcflow/tests/workflows/test_jobscript.py,sha256=9sp1o0g72JZbv2QlOl5v7wCZEFjotxiIKGNUxVaFgaA,724
 hpcflow/tests/workflows/test_workflows.py,sha256=xai6FRtGqG4lStJk6KmsqPUSuvqs9FrsBOxMVALshIs,13400
 hpcflow/viz_demo.ipynb,sha256=1QdnVsk72vihv2L6hOGyk318uEa22ZSgGxQCa7hW2oo,6238
-hpcflow_new2-0.2.0a176.dist-info/METADATA,sha256=Xg97HSWIATWbQGGHQ-ORIDzX1dRhvQ2O0mmtxDluQto,2466
-hpcflow_new2-0.2.0a176.dist-info/WHEEL,sha256=kLuE8m1WYU0Ig0_YEGrXyTtiJvKPpLpDEiChiNyei5Y,88
-hpcflow_new2-0.2.0a176.dist-info/entry_points.txt,sha256=aoGtCnFdfPcXfBdu2zZyMOJoz6fPgdR0elqsgrE-USU,106
-hpcflow_new2-0.2.0a176.dist-info/RECORD,,
+hpcflow_new2-0.2.0a177.dist-info/METADATA,sha256=I1tkrIUSwRwNsKxjcTy7lrpcOhZp9Z3HUGV_Wdoo9qw,2466
+hpcflow_new2-0.2.0a177.dist-info/WHEEL,sha256=kLuE8m1WYU0Ig0_YEGrXyTtiJvKPpLpDEiChiNyei5Y,88
+hpcflow_new2-0.2.0a177.dist-info/entry_points.txt,sha256=aoGtCnFdfPcXfBdu2zZyMOJoz6fPgdR0elqsgrE-USU,106
+hpcflow_new2-0.2.0a177.dist-info/RECORD,,