hpcflow-new2 0.2.0a223__tar.gz → 0.2.0a228__tar.gz

{hpcflow_new2-0.2.0a223 → hpcflow_new2-0.2.0a228}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a223
+Version: 0.2.0a228
 Summary: Computational workflow management
 License: MPL-2.0
 Author: aplowman

hpcflow_new2-0.2.0a228/hpcflow/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0a228"

{hpcflow_new2-0.2.0a223 → hpcflow_new2-0.2.0a228}/hpcflow/sdk/app.py

@@ -3315,12 +3315,13 @@ class BaseApp(metaclass=Singleton):
                     out_item["deleted"] = True
 
                 else:
-                    if status:
-                        status.update(
-                            f"Reading workflow {file_dat_i['path']!r} submission info..."
-                        )
                     with wk_i._store.cache_ctx():
                         sub = wk_i.submissions[file_dat_i["sub_idx"]]
+                        if status:
+                            status.update(
+                                f"Loading workflow {file_dat_i['path']!r} run metadata..."
+                            )
+                        sub.use_EARs_cache = True  # pre-cache EARs of this submission
 
                         all_jobscripts = sub._submission_parts[submit_time_str]
                         out_item["jobscripts"] = all_jobscripts
@@ -3583,7 +3584,7 @@ class BaseApp(metaclass=Singleton):
                         )
                         all_cells["status"] = "/".join(
                             js_state.rich_repr
-                            for js_state in sorted(act_js_states, key=lambda x: x.value)
+                            for js_state in sorted(act_js_states, key=lambda x: x.id)
                         )
                     else:
                         if deleted:
@@ -3645,7 +3646,7 @@ class BaseApp(metaclass=Singleton):
                         all_cells["actions_compact"] = " | ".join(
                             f"[{k.colour}]{k.symbol}[/{k.colour}]:{v}"  # type: ignore
                             for k, v in dict(
-                                sorted(EAR_stat_count.items(), key=lambda x: x[0].value)
+                                sorted(EAR_stat_count.items(), key=lambda x: x[0].id)
                             ).items()
                         )
                     else:

{hpcflow_new2-0.2.0a223 → hpcflow_new2-0.2.0a228}/hpcflow/sdk/cli.py

@@ -4,6 +4,7 @@ Command line interface implementation.
 
 from __future__ import annotations
 import contextlib
+import datetime
 import json
 import os
 import time
@@ -772,6 +773,12 @@ def _make_submission_CLI(app: BaseApp):
     def get_login_nodes(scheduler: SGEPosix):
         pprint(scheduler.get_login_nodes())
 
+    class _DateTimeJSONEncoder(json.JSONEncoder):
+        def default(self, obj):
+            if isinstance(obj, datetime.datetime):
+                return obj.isoformat()
+            return super().default(obj)
+
     @submission.command()
     @click.option(
         "as_json",
@@ -784,7 +791,7 @@ def _make_submission_CLI(app: BaseApp):
         """Print known-submissions information as a formatted Python object."""
         out = app.get_known_submissions(as_json=as_json)
         if as_json:
-            click.echo(json.dumps(out))
+            click.echo(json.dumps(out, cls=_DateTimeJSONEncoder))
         else:
             pprint(out)
 

{hpcflow_new2-0.2.0a223 → hpcflow_new2-0.2.0a228}/hpcflow/sdk/core/actions.py

@@ -3017,6 +3017,7 @@ class Action(JSONLike):
             if all(rule.test(element_iteration=element_iter) for rule in cmd.rules)
         ]
 
+    @TimeIt.decorator
     def get_required_executables(self) -> Iterator[str]:
         """Return executable labels required by this action."""
         for command in self.commands:

{hpcflow_new2-0.2.0a223 → hpcflow_new2-0.2.0a228}/hpcflow/sdk/core/commands.py

@@ -11,6 +11,7 @@ from typing import Any, ClassVar, TYPE_CHECKING
 
 import numpy as np
 
+from hpcflow.sdk.log import TimeIt
 from hpcflow.sdk.typing import hydrate
 from hpcflow.sdk.core.element import ElementResources
 from hpcflow.sdk.core.errors import NoCLIFormatMethodError
@@ -416,6 +417,7 @@ class Command(JSONLike):
     def _extract_executable_labels(cls, cmd_str: str) -> Sequence[str]:
         return cls.__EXE_RE.findall(cmd_str)
 
+    @TimeIt.decorator
     def get_required_executables(self) -> Sequence[str]:
         """Return executable labels required by this command."""
         # an executable label might appear in the `command` or `executable` attribute:

{hpcflow_new2-0.2.0a223 → hpcflow_new2-0.2.0a228}/hpcflow/sdk/core/enums.py

@@ -26,45 +26,85 @@ class ActionScopeType(Enum):
 
 
 @dataclass(frozen=True)
-class _EARStatus:
+class _ReportableStateData:
     """
-    Model of the state of an EARStatus.
+    Model of the state of something that is renderable using a symbol and colour.
+
+    Notes
+    -----
+    This class is used as the value in the the enumeration subclasses
+    :py:class:`EARStatus` and :py:class:`JobscriptElementState`.
+
     """
 
-    _value: int
-    #: Symbol to use when rendering a status.
+    #: ID that distinguishes the state.
+    id: int
+    #: Symbol to use when rendering a state.
     symbol: str
-    #: Colour to use when rendering a status.
+    #: Colour to use when rendering a state.
     colour: str
+    #: Documentation of the meaning of the state.
     __doc__: str = ""
 
 
-class EARStatus(_EARStatus, Enum):
+class _ReportableStateEnum(Enum):
+    """Enumeration superclass for reportable state subclasses with some shared methods."""
+
+    @property
+    def id(self) -> int:
+        """
+        The integer ID associated with this state.
+        """
+        return self.value.id
+
+    @property
+    def colour(self) -> str:
+        """
+        The colour associated with this state.
+        """
+        return self.value.colour
+
+    @property
+    def symbol(self) -> str:
+        """
+        The symbol associated with this state.
+        """
+        return self.value.symbol
+
+    @property
+    def rich_repr(self) -> str:
+        """
+        Rich representation of this enumeration element.
+        """
+        return f"[{self.colour}]{self.symbol}[/{self.colour}]"
+
+
+class EARStatus(_ReportableStateEnum):
     """Enumeration of all possible EAR statuses, and their associated status colour."""
 
     #: Not yet associated with a submission.
-    pending = (
+    pending = _ReportableStateData(
         0,
         ".",
         "grey46",
         "Not yet associated with a submission.",
     )
     #: Associated with a prepared submission that is not yet submitted.
-    prepared = (
+    prepared = _ReportableStateData(
         1,
         ".",
         "grey46",
         "Associated with a prepared submission that is not yet submitted.",
     )
     #: Submitted for execution.
-    submitted = (
+    submitted = _ReportableStateData(
         2,
         ".",
         "grey46",
         "Submitted for execution.",
     )
     #: Executing now.
-    running = (
+    running = _ReportableStateData(
         3,
         "●",
         "dodger_blue1",
@@ -72,7 +112,7 @@ class EARStatus(_EARStatus, Enum):
     )
     #: Not attempted due to a failure of an upstream action on which this depends,
     #: or a loop termination condition being satisfied.
-    skipped = (
+    skipped = _ReportableStateData(
         4,
         "s",
         "dark_orange",
@@ -82,32 +122,27 @@ class EARStatus(_EARStatus, Enum):
         ),
     )
     #: Aborted by the user; downstream actions will be attempted.
-    aborted = (
+    aborted = _ReportableStateData(
         5,
         "A",
         "deep_pink4",
         "Aborted by the user; downstream actions will be attempted.",
     )
     #: Probably exited successfully.
-    success = (
+    success = _ReportableStateData(
         6,
         "■",
         "green3",
         "Probably exited successfully.",
     )
     #: Probably failed.
-    error = (
+    error = _ReportableStateData(
         7,
         "E",
         "red3",
         "Probably failed.",
     )
 
-    @property
-    def value(self) -> int:
-        #: The value of the status.
-        return self._value
-
     @classmethod
     def get_non_running_submitted_states(cls) -> frozenset[EARStatus]:
         """Return the set of all non-running states, excluding those before submission."""
@@ -120,13 +155,6 @@ class EARStatus(_EARStatus, Enum):
             }
         )
 
-    @property
-    def rich_repr(self) -> str:
-        """
-        The rich representation of the value.
-        """
-        return f"[{self.colour}]{self.symbol}[/{self.colour}]"
-
 
 class InputSourceType(Enum):
     """

{hpcflow_new2-0.2.0a223 → hpcflow_new2-0.2.0a228}/hpcflow/sdk/core/parameters.py

@@ -1584,7 +1584,7 @@ class MultiPathSequence(_BaseSequence):
         paths: Sequence[str],
         num_samples: int,
         *,
-        bounds: dict[str, Sequence[float]] | None = None,
+        bounds: dict[str, dict[str, str | Sequence[float]]] | None = None,
         scramble: bool = True,
         strength: int = 1,
         optimization: Literal["random-cd", "lloyd"] | None = None,
@@ -1602,10 +1602,20 @@ class MultiPathSequence(_BaseSequence):
 
         bounds = bounds or {}
 
-        parameter_ranges = np.array([bounds.get(path, [0, 1]) for path in paths]).T
+        scaling = np.asarray(
+            [bounds.get(path, {}).get("scaling", "linear") for path in paths]
+        )
+
+        # extents including defaults for unspecified:
+        all_extents = [bounds.get(path, {}).get("extent", [0, 1]) for path in paths]
 
-        lower_bound = parameter_ranges[0]
-        upper_bound = parameter_ranges[1]
+        # extents accounting for scaling type:
+        extent = np.asarray(
+            [
+                np.log10(all_extents[i]) if scaling[i] == "log" else all_extents[i]
+                for i in range(len(scaling))
+            ]
+        ).T
 
         try:
             sampler = LatinHypercube(**kwargs)
@@ -1614,9 +1624,15 @@ class MultiPathSequence(_BaseSequence):
             kwargs["seed"] = kwargs.pop("rng")
             sampler = LatinHypercube(**kwargs)
 
-        samples = scale(sampler.random(n=num_samples), lower_bound, upper_bound).T
+        samples = scale(
+            sampler.random(n=num_samples), l_bounds=extent[0], u_bounds=extent[1]
+        )
+
+        for i in range(len(scaling)):
+            if scaling[i] == "log":
+                samples[:, i] = 10 ** samples[:, i]
 
-        return samples
+        return samples.T
 
     @classmethod
     def from_latin_hypercube(
@@ -1624,7 +1640,7 @@ class MultiPathSequence(_BaseSequence):
         paths: Sequence[str],
         num_samples: int,
         *,
-        bounds: dict[str, Sequence[float]] | None = None,
+        bounds: dict[str, dict[str, str | Sequence[float]]] | None = None,
         scramble: bool = True,
         strength: int = 1,
         optimization: Literal["random-cd", "lloyd"] | None = None,
@@ -1634,6 +1650,32 @@ class MultiPathSequence(_BaseSequence):
     ) -> Self:
         """
         Generate values from SciPy's latin hypercube sampler: :class:`scipy.stats.qmc.LatinHypercube`.
+
+        Parameters
+        ----------
+        paths : Sequence[str]
+            List of dot-delimited paths within the parameter's nested data structure for which
+            'value' should be set.
+        num_samples : int
+            Number of random hypercube samples to take.
+        bounds : dict[str, dict[str, str  |  Sequence[float]]] | None, optional
+            Bounds dictionary structure which takes a path as a key and returns another dictionary
+            which takes `scaling` and `extent` as keys. `extent` defines the width of the parameter
+            space, and `scaling` defines whether to take logarithmically spaced samples ("log") or not ("linear"). By default,
+            linear scaling and an extent between 0 and 1 is used.
+        scramble : bool, optional
+            See `scipy.stats.qmc.LatinHypercube`, by default True
+        strength : int, optional
+            See 'scipy.stats.qmc.LatinHypercube', by default 1
+        optimization : Literal["random, optional
+            See 'scipy.stats.qmc.LatinHypercube', by default None
+        rng : _type_, optional
+            See 'scipy.stats.qmc.LatinHypercube', by default None
+
+        Returns
+        -------
+        NDArray
+            Array of hypercube samples.
         """
         kwargs = {
             "paths": paths,

{hpcflow_new2-0.2.0a223 → hpcflow_new2-0.2.0a228}/hpcflow/sdk/core/workflow.py

@@ -1786,6 +1786,20 @@ class Workflow(AppAware):
         elem: int
         task: int
 
+    @TimeIt.decorator
+    def __get_elements_by_task_idx(
+        self, element_idx_by_task: dict[int, set[int]]
+    ) -> dict[int, dict[int, Element]]:
+        return {
+            task_idx: {
+                idx: element
+                for idx, element in zip(
+                    elem_indices, self.tasks[task_idx].elements[list(elem_indices)]
+                )
+            }
+            for task_idx, elem_indices in element_idx_by_task.items()
+        }
+
     @TimeIt.decorator
     def get_elements_from_IDs(self, id_lst: Iterable[int]) -> list[Element]:
         """Return element objects from a list of IDs."""
@@ -1800,10 +1814,7 @@ class Workflow(AppAware):
             index_paths.append(Workflow._IndexPath1(elem_idx, task.index))
             element_idx_by_task[task.index].add(elem_idx)
 
-        elements_by_task = {
-            task_idx: {idx: self.tasks[task_idx].elements[idx] for idx in elem_idxes}
-            for task_idx, elem_idxes in element_idx_by_task.items()
-        }
+        elements_by_task = self.__get_elements_by_task_idx(element_idx_by_task)
 
         return [elements_by_task[path.task][path.elem] for path in index_paths]
 
@@ -1832,10 +1843,7 @@ class Workflow(AppAware):
             index_paths.append(Workflow._IndexPath2(iter_idx, elem_idx, task.index))
             element_idx_by_task[task.index].add(elem_idx)
 
-        elements_by_task = {
-            task_idx: {idx: self.tasks[task_idx].elements[idx] for idx in elem_idx}
-            for task_idx, elem_idx in element_idx_by_task.items()
-        }
+        elements_by_task = self.__get_elements_by_task_idx(element_idx_by_task)
 
         return [
             elements_by_task[path.task][path.elem].iterations[path.iter]
@@ -3627,7 +3635,8 @@ class Workflow(AppAware):
         if status:
             status.update("Adding new submission: resolving jobscripts...")
 
-        cache = ObjectCache.build(self, elements=True, iterations=True, runs=True)
+        with self._store.cache_ctx():
+            cache = ObjectCache.build(self, elements=True, iterations=True, runs=True)
 
         sub_obj: Submission = self._app.Submission(
             index=new_idx,
@@ -3719,7 +3728,7 @@ class Workflow(AppAware):
 
         """
         with self._app.config.cached_config():
-            with self.cached_merged_parameters():
+            with self.cached_merged_parameters(), self._store.cache_ctx():
                 js, element_deps = self._resolve_singular_jobscripts(
                     cache, tasks, force_array
                 )

{hpcflow_new2-0.2.0a223 → hpcflow_new2-0.2.0a228}/hpcflow/sdk/persistence/base.py

@@ -1218,12 +1218,15 @@ class PersistentStore(
     @contextlib.contextmanager
     def cache_ctx(self) -> Iterator[None]:
         """Context manager for using the persistent element/iteration/run cache."""
-        self._use_cache = True
-        try:
+        if self._use_cache:
             yield
-        finally:
-            self._use_cache = False
-            self._reset_cache()
+        else:
+            self._use_cache = True
+            try:
+                yield
+            finally:
+                self._use_cache = False
+                self._reset_cache()
 
     @contextlib.contextmanager
     def parameters_metadata_cache(self):

{hpcflow_new2-0.2.0a223 → hpcflow_new2-0.2.0a228}/hpcflow/sdk/submission/enums.py

@@ -6,82 +6,56 @@ from __future__ import annotations
 from dataclasses import dataclass
 from enum import Enum
 
+from hpcflow.sdk.core.enums import _ReportableStateData, _ReportableStateEnum
 
-@dataclass(frozen=True)
-class _JES:
-    """
-    Model of the state of a JobscriptElementState
-    """
-
-    _value: int
-    #: The symbol used to render the state.
-    symbol: str
-    #: The colour used to render the state.
-    colour: str
-    __doc__: str = ""
 
-
-class JobscriptElementState(_JES, Enum):
+class JobscriptElementState(_ReportableStateEnum):
     """Enumeration to convey a particular jobscript element state as reported by the
     scheduler."""
 
     #: Waiting for resource allocation.
-    pending = (
+    pending = _ReportableStateData(
         0,
         "○",
         "yellow",
         "Waiting for resource allocation.",
     )
     #: Waiting for one or more dependencies to finish.
-    waiting = (
+    waiting = _ReportableStateData(
         1,
         "◊",
         "grey46",
         "Waiting for one or more dependencies to finish.",
     )
     #: Executing now.
-    running = (
+    running = _ReportableStateData(
         2,
         "●",
         "dodger_blue1",
         "Executing now.",
     )
     #: Previously submitted but is no longer active.
-    finished = (
+    finished = _ReportableStateData(
         3,
         "■",
         "grey46",
         "Previously submitted but is no longer active.",
     )
     #: Cancelled by the user.
-    cancelled = (
+    cancelled = _ReportableStateData(
         4,
         "C",
         "red3",
         "Cancelled by the user.",
     )
     #: The scheduler reports an error state.
-    errored = (
+    errored = _ReportableStateData(
         5,
         "E",
         "red3",
         "The scheduler reports an error state.",
     )
 
-    @property
-    def value(self) -> int:
-        """
-        The numerical value of this state.
-        """
-        return self._value
-
-    @property
-    def rich_repr(self) -> str:
-        """
-        Rich representation of this enumeration element.
-        """
-        return f"[{self.colour}]{self.symbol}[/{self.colour}]"
-
 
 class SubmissionStatus(Enum):
     """

{hpcflow_new2-0.2.0a223 → hpcflow_new2-0.2.0a228}/hpcflow/sdk/submission/jobscript.py

@@ -901,6 +901,7 @@ class Jobscript(JSONLike):
         )
 
     @property
+    @TimeIt.decorator
     def all_EAR_IDs(self) -> NDArray:
         """Return all run IDs of this jobscripts (across all blocks), removing missing
         run IDs (i.e. -1 values)"""
@@ -912,6 +913,8 @@ class Jobscript(JSONLike):
         """
         Description of EAR information for this jobscript.
         """
+        if self.submission._use_EARs_cache:
+            return [self.submission._EARs_cache[ear_id] for ear_id in self.all_EAR_IDs]
         return self.workflow.get_EARs_from_IDs(self.all_EAR_IDs)
 
     @property