hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

hpcflow/sdk/core/app_aware.py

@@ -0,0 +1,24 @@
+"""
+Utility base class for making classes aware of the overall application context.
+"""
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import ClassVar
+    from ..app import BaseApp
+
+
+class AppAware:
+    """
+    A base class that marks its subclasses as aware of the application.
+
+    Attributes
+    ----------
+    _app: BaseApp
+        A class attribute that holds the application instance.
+    """
+
+    __slots__: ClassVar[tuple[str, ...]] = ()
+    _app: ClassVar[BaseApp]
+    _app_attr: ClassVar[str] = "_app"

hpcflow/sdk/core/cache.py

@@ -2,42 +2,62 @@
 Dependency resolution cache.
 """
 
+from __future__ import annotations
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import Set, Dict
+from itertools import chain
+from typing import TYPE_CHECKING
 
 from hpcflow.sdk.log import TimeIt
 
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+    from typing_extensions import Self
+    from .element import Element, ElementIteration
+    from .actions import ElementActionRun
+    from .workflow import Workflow
+    from ..persistence.base import StoreEAR, StoreElement, StoreElementIter
+
 
 @dataclass
-class DependencyCache:
-    """Class to bulk-retrieve dependencies between elements, iterations, and runs."""
+class ObjectCache:
+    """Class to bulk-retrieve and store elements, iterations, runs and their various
+    dependencies."""
+
+    #: The elements of the workflow that this cache was built from.
+    elements: list[Element] | None = None
+    #: The iterations of the workflow that this cache was built from.
+    iterations: list[ElementIteration] | None = None
+    #: The runs of the workflow that this cache was built from.
+    runs: list[ElementActionRun] | None = None
 
     #: What EARs (by ID) a given EAR depends on.
-    run_dependencies: Dict[int, Set]
+    run_dependencies: dict[int, set[int]] | None = None
     #: What EARs (by ID) are depending on a given EAR.
-    run_dependents: Dict[int, Set]
+    run_dependents: dict[int, set[int]] | None = None
     #: What EARs (by ID) a given iteration depends on.
-    iter_run_dependencies: Dict[int, Set]
+    iter_run_dependencies: dict[int, set[int]] | None = None
     #: What iterations (by ID) a given iteration depends on.
-    iter_iter_dependencies: Dict[int, Set]
+    iter_iter_dependencies: dict[int, set[int]] | None = None
     #: What iterations (by ID) a given element depends on.
-    elem_iter_dependencies: Dict[int, Set]
+    elem_iter_dependencies: dict[int, set[int]] | None = None
     #: What elements (by ID) a given element depends on.
-    elem_elem_dependencies: Dict[int, Set]
+    elem_elem_dependencies: dict[int, set[int]] | None = None
     #: What elements (by ID) are depending on a given element.
-    elem_elem_dependents: Dict[int, Set]
+    elem_elem_dependents: dict[int, set[int]] | None = None
     #: Transitive closure of :py:attr:`elem_elem_dependents`.
-    elem_elem_dependents_rec: Dict[int, Set]
-
-    #: The elements of the workflow that this cache was built from.
-    elements: Dict
-    #: The iterations of the workflow that this cache was built from.
-    iterations: Dict
+    elem_elem_dependents_rec: dict[int, set[int]] | None = None
 
     @classmethod
     @TimeIt.decorator
-    def build(cls, workflow):
+    def build(
+        cls,
+        workflow: Workflow,
+        dependencies: bool = False,
+        elements: bool = False,
+        iterations: bool = False,
+        runs: bool = False,
+    ):
         """
         Build a cache instance.
 
@@ -45,59 +65,108 @@ class DependencyCache:
         ----------
         workflow: ~hpcflow.app.Workflow
             The workflow to build the cache from.
+        dependencies
+            If True, calculate dependencies.
+        elements
+            If True, include elements in the cache.
+        iterations
+            If True, include iterations in the cache.
+        runs
+            If True, include runs in the cache.
+
         """
+        kwargs = {}
+        if dependencies:
+            kwargs.update(cls._get_dependencies(workflow))
+
+        if elements:
+            kwargs["elements"] = workflow.get_all_elements()
+
+        if iterations:
+            kwargs["iterations"] = workflow.get_all_element_iterations()
+
+        if runs:
+            kwargs["runs"] = workflow.get_all_EARs()
+
+        return cls(**kwargs)
+
+    @classmethod
+    @TimeIt.decorator
+    def _get_dependencies(cls, workflow: Workflow):
+        def _get_recursive_deps(elem_id: int, seen_ids: list[int] | None = None):
+            if seen_ids is None:
+                seen_ids = [elem_id]
+            elif elem_id in seen_ids:
+                # stop recursion
+                return set()
+            else:
+                seen_ids.append(elem_id)
+            return set(elem_elem_dependents[elem_id]).union(
+                [
+                    j
+                    for i in elem_elem_dependents[elem_id]
+                    for j in _get_recursive_deps(i, seen_ids)
+                    if j != elem_id
+                ]
+            )
+
         num_iters = workflow.num_element_iterations
         num_elems = workflow.num_elements
         num_runs = workflow.num_EARs
 
-        all_store_runs = workflow._store.get_EARs(list(range(num_runs)))
-        all_store_iters = workflow._store.get_element_iterations(list(range(num_iters)))
-        all_store_elements = workflow._store.get_elements(list(range(num_elems)))
+        all_store_runs: Sequence[StoreEAR] = workflow._store.get_EARs(range(num_runs))
+        all_store_iters: Sequence[
+            StoreElementIter
+        ] = workflow._store.get_element_iterations(range(num_iters))
+        all_store_elements: Sequence[StoreElement] = workflow._store.get_elements(
+            range(num_elems)
+        )
         all_param_sources = workflow.get_all_parameter_sources()
-        all_data_idx = [
+        all_data_idx = (
             {
                 k: v if isinstance(v, list) else [v]
-                for k, v in i.data_idx.items()
-                if k not in ("repeats.",)
+                for k, v in store_ear.data_idx.items()
+                if not k.startswith("repeats.")
             }
-            for i in all_store_runs
-        ]
+            for store_ear in all_store_runs
+        )
 
         # run dependencies and dependents
-        run_dependencies = {}
-        run_dependents = defaultdict(set)
-        for idx, i in enumerate(all_data_idx):
-            run_i_sources = set()
-            for j in i.values():
-                for k in j:
-                    run_k = all_param_sources[k].get("EAR_ID")
-                    if run_k is not None and run_k != idx:
-                        run_i_sources.add(run_k)
+        run_dependencies: dict[int, set[int]] = {}
+        run_dependents: defaultdict[int, set[int]] = defaultdict(set)
+        for idx, dict_i in enumerate(all_data_idx):
+            run_i_sources = set(
+                run_k
+                for dat_idx_k in chain.from_iterable(dict_i.values())
+                if (run_k := all_param_sources[dat_idx_k].get("EAR_ID")) is not None
+                and run_k != idx
+            )
             run_dependencies[idx] = run_i_sources
             for m in run_i_sources:
                 run_dependents[m].add(idx)
 
-        # add missing:
-        for k in range(num_runs):
-            run_dependents[k]
-
-        run_dependents = dict(run_dependents)
+        # add missing and downgrade to dict:
+        for run_idx in range(num_runs):
+            run_dependents[run_idx]
+        run_dependents.default_factory = None
 
         # iteration dependencies
         all_iter_run_IDs = {
-            i.id_: [k for j in i.EAR_IDs.values() for k in j] for i in all_store_iters
+            iter_.id_: tuple(chain.from_iterable((iter_.EAR_IDs or {}).values()))
+            for iter_ in all_store_iters
         }
         # for each iteration, which runs does it depend on?
         iter_run_dependencies = {
-            k: set(j for i in v for j in run_dependencies[i])
+            k: set(j for idx in v for j in run_dependencies[idx])
             for k, v in all_iter_run_IDs.items()
         }
 
         # for each run, which iteration does it belong to?
-        all_run_iter_IDs = {}
-        for iter_ID, run_IDs in all_iter_run_IDs.items():
-            for run_ID in run_IDs:
-                all_run_iter_IDs[run_ID] = iter_ID
+        all_run_iter_IDs = {
+            run_ID: iter_ID
+            for iter_ID, run_IDs in all_iter_run_IDs.items()
+            for run_ID in run_IDs
+        }
 
         # for each iteration, which iterations does it depend on?
         iter_iter_dependencies = {
@@ -105,52 +174,45 @@ class DependencyCache:
             for k, v in iter_run_dependencies.items()
         }
 
-        all_elem_iter_IDs = {i.id_: i.iteration_IDs for i in all_store_elements}
+        all_elem_iter_IDs = {el.id_: el.iteration_IDs for el in all_store_elements}
 
         elem_iter_dependencies = {
-            k: set(j for i in v for j in iter_iter_dependencies[i])
-            for k, v in all_elem_iter_IDs.items()
+            elem_ID: set(j for i in iter_IDs for j in iter_iter_dependencies[i])
+            for elem_ID, iter_IDs in all_elem_iter_IDs.items()
         }
 
         # for each iteration, which element does it belong to?
-        all_iter_elem_IDs = {}
-        for elem_ID, iter_IDs in all_elem_iter_IDs.items():
-            for iter_ID in iter_IDs:
-                all_iter_elem_IDs[iter_ID] = elem_ID
+        all_iter_elem_IDs = {
+            iter_ID: elem_ID
+            for elem_ID, iter_IDs in all_elem_iter_IDs.items()
+            for iter_ID in iter_IDs
+        }
 
         # element dependencies
         elem_elem_dependencies = {
-            k: set(all_iter_elem_IDs[i] for i in v)
-            for k, v in elem_iter_dependencies.items()
+            k: set(all_iter_elem_IDs[i] for i in dep_set)
+            for k, dep_set in elem_iter_dependencies.items()
         }
 
         # for each element, which elements depend on it (directly)?
-        elem_elem_dependents = defaultdict(set)
-        for k, v in elem_elem_dependencies.items():
-            for i in v:
+        elem_elem_dependents: defaultdict[int, set[int]] = defaultdict(set)
+        for k, dep_set in elem_elem_dependencies.items():
+            for i in dep_set:
                 elem_elem_dependents[i].add(k)
 
         # for each element, which elements depend on it (recursively)?
-        elem_elem_dependents_rec = defaultdict(set)
-        for k in list(elem_elem_dependents):
-            for i in elem_elem_dependents[k]:
-                elem_elem_dependents_rec[k].add(i)
-                elem_elem_dependents_rec[k].update(
-                    {m for m in elem_elem_dependents[i] if m != k}
-                )
-
-        # add missing keys:
-        for k in range(num_elems):
-            elem_elem_dependents[k]
-            elem_elem_dependents_rec[k]
-
-        elem_elem_dependents = dict(elem_elem_dependents)
-        elem_elem_dependents_rec = dict(elem_elem_dependents_rec)
-
-        elements = workflow.get_all_elements()
-        iterations = workflow.get_all_element_iterations()
-
-        return cls(
+        elem_elem_dependents_rec: defaultdict[int, set[int]] = defaultdict(set)
+        for i in list(elem_elem_dependents):
+            elem_elem_dependents_rec[i] = _get_recursive_deps(i)
+
+        # add missing keys and downgrade to dict:
+        for elem_idx in range(num_elems):
+            elem_elem_dependents[elem_idx]
+            elem_elem_dependents_rec[elem_idx]
+        elem_elem_dependents.default_factory = None
+        elem_elem_dependents_rec.default_factory = None
+
+        return dict(
             run_dependencies=run_dependencies,
             run_dependents=run_dependents,
             iter_run_dependencies=iter_run_dependencies,
@@ -159,6 +221,4 @@ class DependencyCache:
             elem_elem_dependencies=elem_elem_dependencies,
             elem_elem_dependents=elem_elem_dependents,
             elem_elem_dependents_rec=elem_elem_dependents_rec,
-            elements=elements,
-            iterations=iterations,
         )