hpcflow-new2 0.2.0a158__py3-none-any.whl → 0.2.0a160__py3-none-any.whl

hpcflow/sdk/persistence/base.py

@@ -9,10 +9,11 @@ from datetime import datetime, timezone
 import enum
 import os
 from pathlib import Path
+import re
 import shutil
 import socket
 import time
-from typing import Any, Dict, Iterable, List, Optional, Tuple, TypeVar, Union
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, TypeVar, Union
 
 from hpcflow.sdk.core.utils import (
     flatten,
@@ -107,6 +108,7 @@ class StoreTask:
         """
         return cls(is_pending=False, **task_dat)
 
+    @TimeIt.decorator
     def append_element_IDs(self: AnySTask, pend_IDs: List[int]) -> AnySTask:
         """Return a copy, with additional element IDs."""
         elem_IDs = self.element_IDs[:] + pend_IDs
@@ -164,6 +166,7 @@ class StoreElement:
             "iterations": iters,
         }
 
+    @TimeIt.decorator
     def append_iteration_IDs(self: AnySElement, pend_IDs: List[int]) -> AnySElement:
         """Return a copy, with additional iteration IDs."""
         iter_IDs = self.iteration_IDs[:] + pend_IDs
@@ -234,6 +237,7 @@ class StoreElementIter:
             "loop_idx": self.loop_idx,
         }
 
+    @TimeIt.decorator
     def append_EAR_IDs(
         self: AnySElementIter, pend_IDs: Dict[int, List[int]]
     ) -> AnySElementIter:
@@ -256,6 +260,7 @@ class StoreElementIter:
             EARs_initialised=self.EARs_initialised,
         )
 
+    @TimeIt.decorator
     def update_loop_idx(
         self: AnySElementIter, loop_idx: Dict[str, int]
     ) -> AnySElementIter:
@@ -273,6 +278,7 @@ class StoreElementIter:
             loop_idx=loop_idx_new,
         )
 
+    @TimeIt.decorator
     def set_EARs_initialised(self: AnySElementIter) -> AnySElementIter:
         """Return a copy with `EARs_initialised` set to `True`."""
         return self.__class__(
@@ -378,6 +384,7 @@ class StoreEAR:
             "run_hostname": self.run_hostname,
         }
 
+    @TimeIt.decorator
     def update(
         self,
         submission_idx: Optional[int] = None,
@@ -655,6 +662,10 @@ class PersistentStore(ABC):
         self._resources_in_use = set()
         self._in_batch_mode = False
 
+        self._use_cache = False
+        self._cache = None
+        self._reset_cache()
+
     @property
     def logger(self):
         return self.app.persistence_logger
@@ -672,6 +683,70 @@ class PersistentStore(ABC):
         """Does this store support workflow submission?"""
         return self.fs.__class__.__name__ == "LocalFileSystem"
 
+    @property
+    def use_cache(self):
+        return self._use_cache
+
+    @property
+    def task_cache(self):
+        """Cache for persistent tasks."""
+        return self._cache["tasks"]
+
+    @property
+    def element_cache(self):
+        """Cache for persistent elements."""
+        return self._cache["elements"]
+
+    @property
+    def element_iter_cache(self):
+        """Cache for persistent element iterations."""
+        return self._cache["element_iters"]
+
+    @property
+    def EAR_cache(self):
+        """Cache for persistent EARs."""
+        return self._cache["EARs"]
+
+    @property
+    def num_tasks_cache(self):
+        """Cache for number of persistent tasks."""
+        return self._cache["num_tasks"]
+
+    @property
+    def param_sources_cache(self):
+        """Cache for persistent parameter sources."""
+        return self._cache["param_sources"]
+
+    @property
+    def parameter_cache(self):
+        """Cache for persistent parameters."""
+        return self._cache["parameters"]
+
+    @num_tasks_cache.setter
+    def num_tasks_cache(self, value):
+        self._cache["num_tasks"] = value
+
+    def _reset_cache(self):
+        self._cache = {
+            "tasks": {},
+            "elements": {},
+            "element_iters": {},
+            "EARs": {},
+            "param_sources": {},
+            "num_tasks": None,
+            "parameters": {},
+        }
+
+    @contextlib.contextmanager
+    def cache_ctx(self):
+        """Context manager for using the persistent element/iteration/run cache."""
+        self._use_cache = True
+        try:
+            yield
+        finally:
+            self._use_cache = False
+            self._reset_cache()
+
     @staticmethod
     def prepare_test_store_from_spec(task_spec):
         """Generate a valid store from a specification in terms of nested
@@ -872,6 +947,7 @@ class PersistentStore(ABC):
         if save:
             self.save()
 
+    @TimeIt.decorator
     def add_submission(self, sub_idx: int, sub_js: Dict, save: bool = True):
         """Add a new submission."""
         self.logger.debug(f"Adding store submission.")
@@ -964,6 +1040,7 @@ class PersistentStore(ABC):
         if save:
             self.save()
 
+    @TimeIt.decorator
     def set_EAR_submission_index(
         self, EAR_ID: int, sub_idx: int, save: bool = True
     ) -> None:
@@ -973,9 +1050,7 @@ class PersistentStore(ABC):
 
     def set_EAR_start(self, EAR_ID: int, save: bool = True) -> datetime:
         dt = datetime.utcnow()
-        snapshot = JSONLikeDirSnapShot()
-        snapshot.take(".")
-        ss_js = snapshot.to_json_like()
+        ss_js = self.app.RunDirAppFiles.take_snapshot()
         run_hostname = socket.gethostname()
         self._pending.set_EAR_starts[EAR_ID] = (dt, ss_js, run_hostname)
         if save:
@@ -987,9 +1062,7 @@ class PersistentStore(ABC):
     ) -> datetime:
         # TODO: save output files
         dt = datetime.utcnow()
-        snapshot = JSONLikeDirSnapShot()
-        snapshot.take(".")
-        ss_js = snapshot.to_json_like()
+        ss_js = self.app.RunDirAppFiles.take_snapshot()
         self._pending.set_EAR_ends[EAR_ID] = (dt, ss_js, exit_code, success)
         if save:
             self.save()
@@ -1245,6 +1318,7 @@ class PersistentStore(ABC):
     def _get_task_id_to_idx_map(self) -> Dict[int, int]:
         return {i.id_: i.index for i in self.get_tasks()}
 
+    @TimeIt.decorator
     def get_task(self, task_idx: int) -> AnySTask:
         return self.get_tasks()[task_idx]
 
@@ -1287,10 +1361,10 @@ class PersistentStore(ABC):
 
         return self._process_retrieved_tasks(tasks)
 
+    @TimeIt.decorator
     def get_tasks(self) -> List[AnySTask]:
         """Retrieve all tasks, including pending."""
-
-        tasks = self._get_persistent_tasks()
+        tasks = self._get_persistent_tasks(range(self._get_num_persistent_tasks()))
         tasks.update({k: v for k, v in self._pending.add_tasks.items()})
 
         # order by index:
@@ -1326,6 +1400,7 @@ class PersistentStore(ABC):
 
         return self._process_retrieved_loops(loops)
 
+    @TimeIt.decorator
     def get_submissions(self) -> Dict[int, Dict]:
         """Retrieve all submissions, including pending."""
 
@@ -1337,6 +1412,7 @@ class PersistentStore(ABC):
 
         return subs
 
+    @TimeIt.decorator
     def get_submissions_by_ID(self, id_lst: Iterable[int]) -> Dict[int, Dict]:
         # separate pending and persistent IDs:
         id_set = set(id_lst)
@@ -1352,6 +1428,7 @@ class PersistentStore(ABC):
 
         return subs
 
+    @TimeIt.decorator
     def get_elements(self, id_lst: Iterable[int]) -> List[AnySElement]:
         self.logger.debug(f"PersistentStore.get_elements: id_lst={id_lst!r}")
 
@@ -1378,6 +1455,7 @@ class PersistentStore(ABC):
 
         return elems_new
 
+    @TimeIt.decorator
     def get_element_iterations(self, id_lst: Iterable[int]) -> List[AnySElementIter]:
         self.logger.debug(f"PersistentStore.get_element_iterations: id_lst={id_lst!r}")
 
@@ -1413,6 +1491,7 @@ class PersistentStore(ABC):
 
         return iters_new
 
+    @TimeIt.decorator
     def get_EARs(self, id_lst: Iterable[int]) -> List[AnySEAR]:
         self.logger.debug(f"PersistentStore.get_EARs: id_lst={id_lst!r}")
 
@@ -1457,10 +1536,51 @@ class PersistentStore(ABC):
 
         return EARs_new
 
+    @TimeIt.decorator
+    def _get_cached_persistent_items(
+        self, id_lst: Iterable[int], cache: Dict
+    ) -> Tuple[Dict[int, Any], List[int]]:
+        id_lst = list(id_lst)
+        if self.use_cache:
+            id_set = set(id_lst)
+            all_cached = set(cache.keys())
+            id_cached = id_set.intersection(all_cached)
+            id_non_cached = list(id_set.difference(all_cached))
+            items = {k: cache[k] for k in id_cached}
+        else:
+            items = {}
+            id_non_cached = id_lst
+        return items, id_non_cached
+
+    def _get_cached_persistent_EARs(
+        self, id_lst: Iterable[int]
+    ) -> Tuple[Dict[int, AnySEAR], List[int]]:
+        return self._get_cached_persistent_items(id_lst, self.EAR_cache)
+
+    def _get_cached_persistent_element_iters(
+        self, id_lst: Iterable[int]
+    ) -> Tuple[Dict[int, AnySEAR], List[int]]:
+        return self._get_cached_persistent_items(id_lst, self.element_iter_cache)
+
+    def _get_cached_persistent_elements(
+        self, id_lst: Iterable[int]
+    ) -> Tuple[Dict[int, AnySEAR], List[int]]:
+        return self._get_cached_persistent_items(id_lst, self.element_cache)
+
+    def _get_cached_persistent_tasks(self, id_lst: Iterable[int]):
+        return self._get_cached_persistent_items(id_lst, self.task_cache)
+
+    def _get_cached_persistent_param_sources(self, id_lst: Iterable[int]):
+        return self._get_cached_persistent_items(id_lst, self.param_sources_cache)
+
+    def _get_cached_persistent_parameters(self, id_lst: Iterable[int]):
+        return self._get_cached_persistent_items(id_lst, self.parameter_cache)
+
     def get_EAR_skipped(self, EAR_ID: int) -> bool:
         self.logger.debug(f"PersistentStore.get_EAR_skipped: EAR_ID={EAR_ID!r}")
         return self.get_EARs([EAR_ID])[0].skip
 
+    @TimeIt.decorator
     def get_parameters(
         self,
         id_lst: Iterable[int],
@@ -1487,6 +1607,7 @@ class PersistentStore(ABC):
 
         return params
 
+    @TimeIt.decorator
     def get_parameter_set_statuses(self, id_lst: Iterable[int]) -> List[bool]:
         # separate pending and persistent IDs:
         id_set = set(id_lst)
@@ -1500,6 +1621,7 @@ class PersistentStore(ABC):
         # order as requested:
         return [set_status[id_] for id_ in id_lst]
 
+    @TimeIt.decorator
     def get_parameter_sources(self, id_lst: Iterable[int]) -> List[Dict]:
         # separate pending and persistent IDs:
         id_set = set(id_lst)
@@ -1523,15 +1645,23 @@ class PersistentStore(ABC):
 
         return src_new
 
-    def get_task_elements(self, task_id, idx_sel: slice) -> List[Dict]:
-        """Get element data by an index slice within a given task.
+    @TimeIt.decorator
+    def get_task_elements(
+        self,
+        task_id,
+        idx_lst: Optional[Iterable[int]] = None,
+    ) -> List[Dict]:
+        """Get element data by an indices within a given task.
 
         Element iterations and EARs belonging to the elements are included.
 
         """
 
         all_elem_IDs = self.get_task(task_id).element_IDs
-        req_IDs = all_elem_IDs[idx_sel]
+        if idx_lst is None:
+            req_IDs = all_elem_IDs
+        else:
+            req_IDs = [all_elem_IDs[i] for i in idx_lst]
         store_elements = self.get_elements(req_IDs)
         iter_IDs = [i.iteration_IDs for i in store_elements]
         iter_IDs_flat, iter_IDs_lens = flatten(iter_IDs)

hpcflow/sdk/persistence/json.py

@@ -229,9 +229,10 @@ class JSONPersistentStore(PersistentStore):
         with self.using_resource("metadata", action="update") as md:
             md["runs"].extend(i.encode(self.ts_fmt) for i in EARs)
 
-    def _update_EAR_submission_index(self, EAR_id: int, sub_idx: int):
+    def _update_EAR_submission_indices(self, sub_indices: Dict[int, int]):
         with self.using_resource("metadata", action="update") as md:
-            md["runs"][EAR_id]["submission_idx"] = sub_idx
+            for EAR_ID_i, sub_idx_i in sub_indices.items():
+                md["runs"][EAR_ID_i]["submission_idx"] = sub_idx_i
 
     def _update_EAR_start(self, EAR_id: int, s_time: datetime, s_snap: Dict, s_hn: str):
         with self.using_resource("metadata", action="update") as md:
@@ -264,20 +265,6 @@ class JSONPersistentStore(PersistentStore):
                 params["data"][str(param_i.id_)] = param_i.encode()
                 params["sources"][str(param_i.id_)] = param_i.source
 
-    def _set_parameter_value(self, param_id: int, value: Any, is_file: bool):
-        """Set an unset persistent parameter."""
-
-        # the `decode` call in `_get_persistent_parameters` should be quick:
-        param = self._get_persistent_parameters([param_id])[param_id]
-        if is_file:
-            param = param.set_file(value)
-        else:
-            param = param.set_data(value)
-
-        with self.using_resource("parameters", "update") as params:
-            # no need to update sources array:
-            params["data"][str(param_id)] = param.encode()
-
     def _set_parameter_values(self, set_parameters: Dict[int, Tuple[Any, bool]]):
         """Set multiple unset persistent parameters."""
         param_ids = list(set_parameters.keys())
@@ -310,8 +297,13 @@ class JSONPersistentStore(PersistentStore):
 
     def _get_num_persistent_tasks(self) -> int:
         """Get the number of persistent tasks."""
-        with self.using_resource("metadata", action="read") as md:
-            return len(md["tasks"])
+        if self.num_tasks_cache is not None:
+            num = self.num_tasks_cache
+        else:
+            with self.using_resource("metadata", action="read") as md:
+                num = len(md["tasks"])
+                self.num_tasks_cache = num
+        return num
 
     def _get_num_persistent_loops(self) -> int:
         """Get the number of persistent loops."""
@@ -386,16 +378,18 @@ class JSONPersistentStore(PersistentStore):
         with self.using_resource("metadata", "read") as md:
             return md["template"]
 
-    def _get_persistent_tasks(
-        self, id_lst: Optional[Iterable[int]] = None
-    ) -> Dict[int, StoreTask]:
-        with self.using_resource("metadata", action="read") as md:
-            task_dat = {
-                i["id_"]: StoreTask.decode({**i, "index": idx})
-                for idx, i in enumerate(md["tasks"])
-                if id_lst is None or i["id_"] in id_lst
-            }
-        return task_dat
+    def _get_persistent_tasks(self, id_lst: Iterable[int]) -> Dict[int, StoreTask]:
+        tasks, id_lst = self._get_cached_persistent_tasks(id_lst)
+        if id_lst:
+            with self.using_resource("metadata", action="read") as md:
+                new_tasks = {
+                    i["id_"]: StoreTask.decode({**i, "index": idx})
+                    for idx, i in enumerate(md["tasks"])
+                    if id_lst is None or i["id_"] in id_lst
+                }
+                self.task_cache.update(new_tasks)
+                tasks.update(new_tasks)
+        return tasks
 
     def _get_persistent_loops(self, id_lst: Optional[Iterable[int]] = None):
         with self.using_resource("metadata", "read") as md:
@@ -428,54 +422,81 @@ class JSONPersistentStore(PersistentStore):
         return subs_dat
 
     def _get_persistent_elements(self, id_lst: Iterable[int]) -> Dict[int, StoreElement]:
-        # could convert `id_lst` to e.g. slices if more efficient for a given store
-        with self.using_resource("metadata", action="read") as md:
-            try:
-                elem_dat = {i: md["elements"][i] for i in id_lst}
-            except KeyError:
-                raise MissingStoreElementError(id_lst) from None
-            return {k: StoreElement.decode(v) for k, v in elem_dat.items()}
+        elems, id_lst = self._get_cached_persistent_elements(id_lst)
+        if id_lst:
+            # could convert `id_lst` to e.g. slices if more efficient for a given store
+            with self.using_resource("metadata", action="read") as md:
+                try:
+                    elem_dat = {i: md["elements"][i] for i in id_lst}
+                except KeyError:
+                    raise MissingStoreElementError(id_lst) from None
+                new_elems = {k: StoreElement.decode(v) for k, v in elem_dat.items()}
+                self.element_cache.update(new_elems)
+                elems.update(new_elems)
+        return elems
 
     def _get_persistent_element_iters(
         self, id_lst: Iterable[int]
     ) -> Dict[int, StoreElementIter]:
-        with self.using_resource("metadata", action="read") as md:
-            try:
-                iter_dat = {i: md["iters"][i] for i in id_lst}
-            except KeyError:
-                raise MissingStoreElementIterationError(id_lst) from None
-            return {k: StoreElementIter.decode(v) for k, v in iter_dat.items()}
+        iters, id_lst = self._get_cached_persistent_element_iters(id_lst)
+        if id_lst:
+            with self.using_resource("metadata", action="read") as md:
+                try:
+                    iter_dat = {i: md["iters"][i] for i in id_lst}
+                except KeyError:
+                    raise MissingStoreElementIterationError(id_lst) from None
+                new_iters = {k: StoreElementIter.decode(v) for k, v in iter_dat.items()}
+                self.element_iter_cache.update(new_iters)
+                iters.update(new_iters)
+        return iters
 
     def _get_persistent_EARs(self, id_lst: Iterable[int]) -> Dict[int, StoreEAR]:
-        with self.using_resource("metadata", action="read") as md:
-            try:
-                EAR_dat = {i: md["runs"][i] for i in id_lst}
-            except KeyError:
-                raise MissingStoreEARError(id_lst) from None
-            return {k: StoreEAR.decode(v, self.ts_fmt) for k, v in EAR_dat.items()}
+        runs, id_lst = self._get_cached_persistent_EARs(id_lst)
+        if id_lst:
+            with self.using_resource("metadata", action="read") as md:
+                try:
+                    EAR_dat = {i: md["runs"][i] for i in id_lst}
+                except KeyError:
+                    raise MissingStoreEARError(id_lst) from None
+                new_runs = {
+                    k: StoreEAR.decode(v, self.ts_fmt) for k, v in EAR_dat.items()
+                }
+                self.EAR_cache.update(new_runs)
+                runs.update(new_runs)
+        return runs
 
     def _get_persistent_parameters(
         self,
         id_lst: Iterable[int],
     ) -> Dict[int, StoreParameter]:
-        with self.using_resource("parameters", "read") as params:
-            try:
-                param_dat = {i: params["data"][str(i)] for i in id_lst}
-                src_dat = {i: params["sources"][str(i)] for i in id_lst}
-            except KeyError:
-                raise MissingParameterData(id_lst) from None
-
-        return {
-            k: StoreParameter.decode(id_=k, data=v, source=src_dat[k])
-            for k, v in param_dat.items()
-        }
+        params, id_lst = self._get_cached_persistent_parameters(id_lst)
+        if id_lst:
+            with self.using_resource("parameters", "read") as params:
+                try:
+                    param_dat = {i: params["data"][str(i)] for i in id_lst}
+                    src_dat = {i: params["sources"][str(i)] for i in id_lst}
+                except KeyError:
+                    raise MissingParameterData(id_lst) from None
+
+            new_params = {
+                k: StoreParameter.decode(id_=k, data=v, source=src_dat[k])
+                for k, v in param_dat.items()
+            }
+            self.parameter_cache.update(new_params)
+            params.update(new_params)
+        return params
 
     def _get_persistent_param_sources(self, id_lst: Iterable[int]) -> Dict[int, Dict]:
-        with self.using_resource("parameters", "read") as params:
-            try:
-                return {i: params["sources"][str(i)] for i in id_lst}
-            except KeyError:
-                raise MissingParameterData(id_lst) from None
+        sources, id_lst = self._get_cached_persistent_param_sources(id_lst)
+        if id_lst:
+            with self.using_resource("parameters", "read") as params:
+                try:
+                    new_sources = {i: params["sources"][str(i)] for i in id_lst}
+                except KeyError:
+                    raise MissingParameterData(id_lst) from None
+            self.param_sources_cache.update(new_sources)
+            sources.update(new_sources)
+        return sources
 
     def _get_persistent_parameter_set_status(
         self, id_lst: Iterable[int]

hpcflow/sdk/persistence/pending.py

@@ -142,6 +142,7 @@ class PendingChanges:
             task_ids = list(self.add_tasks.keys())
             self.logger.debug(f"commit: adding pending tasks with IDs: {task_ids!r}")
             self.store._append_tasks(tasks)
+            self.store.num_tasks_cache = None  # invalidate cache
             # pending element IDs that belong to pending tasks are now committed:
             self.add_elem_IDs = {
                 k: v for k, v in self.add_elem_IDs.items() if k not in task_ids
@@ -187,6 +188,7 @@ class PendingChanges:
                 f"commit: adding pending element IDs to task {task_ID!r}: {elem_IDs!r}."
             )
             self.store._append_task_element_IDs(task_ID, elem_IDs)
+            self.store.task_cache.pop(task_ID, None)  # invalidate cache
         self.clear_add_elem_IDs()
 
     @TimeIt.decorator
@@ -219,6 +221,7 @@ class PendingChanges:
                 f"{iter_IDs!r}."
             )
             self.store._append_elem_iter_IDs(elem_ID, iter_IDs)
+            self.store.element_cache.pop(elem_ID, None)  # invalidate cache
         self.clear_add_elem_iter_IDs()
 
     @TimeIt.decorator
@@ -250,6 +253,7 @@ class PendingChanges:
             )
             for act_idx, EAR_IDs in act_EAR_IDs.items():
                 self.store._append_elem_iter_EAR_IDs(iter_ID, act_idx, EAR_IDs)
+            self.store.element_iter_cache.pop(iter_ID, None)  # invalidate cache
         self.clear_add_elem_iter_EAR_IDs()
 
     @TimeIt.decorator
@@ -286,19 +290,21 @@ class PendingChanges:
             )
             # TODO: could be batched up?
             for i in iter_ids:
-                self._store._update_elem_iter_EARs_initialised(i)
+                self.store._update_elem_iter_EARs_initialised(i)
+                self.store.element_iter_cache.pop(i, None)  # invalidate cache
         self.clear_set_EARs_initialised()
 
     @TimeIt.decorator
     def commit_EAR_submission_indices(self) -> None:
-        # TODO: could be batched up?
-        for EAR_id, sub_idx in self.set_EAR_submission_indices.items():
+        if self.set_EAR_submission_indices:
             self.logger.debug(
-                f"commit: adding pending submission index ({sub_idx!r}) to EAR ID "
-                f"{EAR_id!r}."
+                f"commit: updating submission indices: "
+                f"{self.set_EAR_submission_indices!r}."
             )
-            self.store._update_EAR_submission_index(EAR_id, sub_idx)
-        self.clear_set_EAR_submission_indices()
+            self.store._update_EAR_submission_indices(self.set_EAR_submission_indices)
+            for EAR_ID_i in self.set_EAR_submission_indices.keys():
+                self.store.EAR_cache.pop(EAR_ID_i, None)  # invalidate cache
+            self.clear_set_EAR_submission_indices()
 
     @TimeIt.decorator
     def commit_EAR_starts(self) -> None:
@@ -309,6 +315,7 @@ class PendingChanges:
                 f"({hostname!r}), and directory snapshot to EAR ID {EAR_id!r}."
             )
             self.store._update_EAR_start(EAR_id, time, snap, hostname)
+            self.store.EAR_cache.pop(EAR_id, None)  # invalidate cache
         self.clear_set_EAR_starts()
 
     @TimeIt.decorator
@@ -320,6 +327,7 @@ class PendingChanges:
                 f"exit code ({ext!r}), and success status {suc!r} to EAR ID {EAR_id!r}."
             )
             self.store._update_EAR_end(EAR_id, time, snap, ext, suc)
+            self.store.EAR_cache.pop(EAR_id, None)  # invalidate cache
         self.clear_set_EAR_ends()
 
     @TimeIt.decorator
@@ -328,6 +336,7 @@ class PendingChanges:
         for EAR_id in self.set_EAR_skips:
             self.logger.debug(f"commit: setting EAR ID {EAR_id!r} as skipped.")
             self.store._update_EAR_skip(EAR_id)
+            self.store.EAR_cache.pop(EAR_id, None)  # invalidate cache
         self.clear_set_EAR_skips()
 
     @TimeIt.decorator
@@ -353,6 +362,8 @@ class PendingChanges:
             param_ids = list(self.set_parameters.keys())
             self.logger.debug(f"commit: setting values of parameter IDs {param_ids!r}.")
             self.store._set_parameter_values(self.set_parameters)
+            for id_i in param_ids:
+                self.store.parameter_cache.pop(id_i, None)
 
         self.clear_set_parameters()
 
@@ -378,6 +389,8 @@ class PendingChanges:
             param_ids = list(self.update_param_sources.keys())
             self.logger.debug(f"commit: updating sources of parameter IDs {param_ids!r}.")
             self.store._update_parameter_sources(self.update_param_sources)
+            for id_i in param_ids:
+                self.store.param_sources_cache.pop(id_i, None)  # invalidate cache
             self.clear_update_param_sources()
 
     @TimeIt.decorator
@@ -389,6 +402,7 @@ class PendingChanges:
                 f"{loop_idx!r}."
             )
             self.store._update_loop_index(iter_ID, loop_idx)
+            self.store.element_iter_cache.pop(iter_ID, None)  # invalidate cache
         self.clear_update_loop_indices()
 
     @TimeIt.decorator

hpcflow/sdk/persistence/utils.py

@@ -1,10 +1,11 @@
 from getpass import getpass
-from paramiko.ssh_exception import SSHException
 
 from hpcflow.sdk.core.errors import WorkflowNotFoundError
 
 
 def ask_pw_on_auth_exc(f, *args, add_pw_to=None, **kwargs):
+    from paramiko.ssh_exception import SSHException
+
     try:
         out = f(*args, **kwargs)
         pw = None