hpcflow-new2 0.2.0a222__tar.gz → 0.2.0a227__tar.gz

{hpcflow_new2-0.2.0a222 → hpcflow_new2-0.2.0a227}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a222
+Version: 0.2.0a227
 Summary: Computational workflow management
 License: MPL-2.0
 Author: aplowman

hpcflow_new2-0.2.0a227/hpcflow/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0a227"

{hpcflow_new2-0.2.0a222 → hpcflow_new2-0.2.0a227}/hpcflow/sdk/app.py

@@ -3315,12 +3315,13 @@ class BaseApp(metaclass=Singleton):
                     out_item["deleted"] = True
 
                 else:
-                    if status:
-                        status.update(
-                            f"Reading workflow {file_dat_i['path']!r} submission info..."
-                        )
                     with wk_i._store.cache_ctx():
                         sub = wk_i.submissions[file_dat_i["sub_idx"]]
+                        if status:
+                            status.update(
+                                f"Loading workflow {file_dat_i['path']!r} run metadata..."
+                            )
+                        sub.use_EARs_cache = True  # pre-cache EARs of this submission
 
                         all_jobscripts = sub._submission_parts[submit_time_str]
                         out_item["jobscripts"] = all_jobscripts
@@ -3583,7 +3584,7 @@ class BaseApp(metaclass=Singleton):
                         )
                         all_cells["status"] = "/".join(
                             js_state.rich_repr
-                            for js_state in sorted(act_js_states, key=lambda x: x.value)
+                            for js_state in sorted(act_js_states, key=lambda x: x.id)
                         )
                     else:
                         if deleted:
@@ -3645,7 +3646,7 @@ class BaseApp(metaclass=Singleton):
                         all_cells["actions_compact"] = " | ".join(
                             f"[{k.colour}]{k.symbol}[/{k.colour}]:{v}"  # type: ignore
                             for k, v in dict(
-                                sorted(EAR_stat_count.items(), key=lambda x: x[0].value)
+                                sorted(EAR_stat_count.items(), key=lambda x: x[0].id)
                             ).items()
                         )
                     else:

{hpcflow_new2-0.2.0a222 → hpcflow_new2-0.2.0a227}/hpcflow/sdk/cli.py

@@ -4,6 +4,7 @@ Command line interface implementation.
 
 from __future__ import annotations
 import contextlib
+import datetime
 import json
 import os
 import time
@@ -772,6 +773,12 @@ def _make_submission_CLI(app: BaseApp):
     def get_login_nodes(scheduler: SGEPosix):
         pprint(scheduler.get_login_nodes())
 
+    class _DateTimeJSONEncoder(json.JSONEncoder):
+        def default(self, obj):
+            if isinstance(obj, datetime.datetime):
+                return obj.isoformat()
+            return super().default(obj)
+
     @submission.command()
     @click.option(
         "as_json",
@@ -784,7 +791,7 @@ def _make_submission_CLI(app: BaseApp):
         """Print known-submissions information as a formatted Python object."""
         out = app.get_known_submissions(as_json=as_json)
         if as_json:
-            click.echo(json.dumps(out))
+            click.echo(json.dumps(out, cls=_DateTimeJSONEncoder))
         else:
             pprint(out)
 

{hpcflow_new2-0.2.0a222 → hpcflow_new2-0.2.0a227}/hpcflow/sdk/core/enums.py

@@ -26,45 +26,85 @@ class ActionScopeType(Enum):
 
 
 @dataclass(frozen=True)
-class _EARStatus:
+class _ReportableStateData:
     """
-    Model of the state of an EARStatus.
+    Model of the state of something that is renderable using a symbol and colour.
+
+    Notes
+    -----
+    This class is used as the value in the the enumeration subclasses
+    :py:class:`EARStatus` and :py:class:`JobscriptElementState`.
+
     """
 
-    _value: int
-    #: Symbol to use when rendering a status.
+    #: ID that distinguishes the state.
+    id: int
+    #: Symbol to use when rendering a state.
     symbol: str
-    #: Colour to use when rendering a status.
+    #: Colour to use when rendering a state.
     colour: str
+    #: Documentation of the meaning of the state.
     __doc__: str = ""
 
 
-class EARStatus(_EARStatus, Enum):
+class _ReportableStateEnum(Enum):
+    """Enumeration superclass for reportable state subclasses with some shared methods."""
+
+    @property
+    def id(self) -> int:
+        """
+        The integer ID associated with this state.
+        """
+        return self.value.id
+
+    @property
+    def colour(self) -> str:
+        """
+        The colour associated with this state.
+        """
+        return self.value.colour
+
+    @property
+    def symbol(self) -> str:
+        """
+        The symbol associated with this state.
+        """
+        return self.value.symbol
+
+    @property
+    def rich_repr(self) -> str:
+        """
+        Rich representation of this enumeration element.
+        """
+        return f"[{self.colour}]{self.symbol}[/{self.colour}]"
+
+
+class EARStatus(_ReportableStateEnum):
     """Enumeration of all possible EAR statuses, and their associated status colour."""
 
     #: Not yet associated with a submission.
-    pending = (
+    pending = _ReportableStateData(
         0,
         ".",
         "grey46",
         "Not yet associated with a submission.",
     )
     #: Associated with a prepared submission that is not yet submitted.
-    prepared = (
+    prepared = _ReportableStateData(
         1,
         ".",
         "grey46",
         "Associated with a prepared submission that is not yet submitted.",
     )
     #: Submitted for execution.
-    submitted = (
+    submitted = _ReportableStateData(
         2,
         ".",
         "grey46",
         "Submitted for execution.",
     )
     #: Executing now.
-    running = (
+    running = _ReportableStateData(
         3,
         "●",
         "dodger_blue1",
@@ -72,7 +112,7 @@ class EARStatus(_EARStatus, Enum):
     )
     #: Not attempted due to a failure of an upstream action on which this depends,
     #: or a loop termination condition being satisfied.
-    skipped = (
+    skipped = _ReportableStateData(
         4,
         "s",
         "dark_orange",
@@ -82,32 +122,27 @@ class EARStatus(_EARStatus, Enum):
         ),
     )
     #: Aborted by the user; downstream actions will be attempted.
-    aborted = (
+    aborted = _ReportableStateData(
         5,
         "A",
         "deep_pink4",
         "Aborted by the user; downstream actions will be attempted.",
     )
     #: Probably exited successfully.
-    success = (
+    success = _ReportableStateData(
         6,
         "■",
         "green3",
         "Probably exited successfully.",
     )
     #: Probably failed.
-    error = (
+    error = _ReportableStateData(
         7,
         "E",
         "red3",
         "Probably failed.",
     )
 
-    @property
-    def value(self) -> int:
-        #: The value of the status.
-        return self._value
-
     @classmethod
     def get_non_running_submitted_states(cls) -> frozenset[EARStatus]:
         """Return the set of all non-running states, excluding those before submission."""
@@ -120,13 +155,6 @@ class EARStatus(_EARStatus, Enum):
             }
         )
 
-    @property
-    def rich_repr(self) -> str:
-        """
-        The rich representation of the value.
-        """
-        return f"[{self.colour}]{self.symbol}[/{self.colour}]"
-
 
 class InputSourceType(Enum):
     """

{hpcflow_new2-0.2.0a222 → hpcflow_new2-0.2.0a227}/hpcflow/sdk/core/parameters.py

@@ -1584,7 +1584,7 @@ class MultiPathSequence(_BaseSequence):
         paths: Sequence[str],
         num_samples: int,
         *,
-        bounds: dict[str, Sequence[float]] | None = None,
+        bounds: dict[str, dict[str, str | Sequence[float]]] | None = None,
         scramble: bool = True,
         strength: int = 1,
         optimization: Literal["random-cd", "lloyd"] | None = None,
@@ -1602,10 +1602,20 @@ class MultiPathSequence(_BaseSequence):
 
         bounds = bounds or {}
 
-        parameter_ranges = np.array([bounds.get(path, [0, 1]) for path in paths]).T
+        scaling = np.asarray(
+            [bounds.get(path, {}).get("scaling", "linear") for path in paths]
+        )
+
+        # extents including defaults for unspecified:
+        all_extents = [bounds.get(path, {}).get("extent", [0, 1]) for path in paths]
 
-        lower_bound = parameter_ranges[0]
-        upper_bound = parameter_ranges[1]
+        # extents accounting for scaling type:
+        extent = np.asarray(
+            [
+                np.log10(all_extents[i]) if scaling[i] == "log" else all_extents[i]
+                for i in range(len(scaling))
+            ]
+        ).T
 
         try:
             sampler = LatinHypercube(**kwargs)
@@ -1614,9 +1624,15 @@ class MultiPathSequence(_BaseSequence):
             kwargs["seed"] = kwargs.pop("rng")
             sampler = LatinHypercube(**kwargs)
 
-        samples = scale(sampler.random(n=num_samples), lower_bound, upper_bound).T
+        samples = scale(
+            sampler.random(n=num_samples), l_bounds=extent[0], u_bounds=extent[1]
+        )
+
+        for i in range(len(scaling)):
+            if scaling[i] == "log":
+                samples[:, i] = 10 ** samples[:, i]
 
-        return samples
+        return samples.T
 
     @classmethod
     def from_latin_hypercube(
@@ -1624,7 +1640,7 @@ class MultiPathSequence(_BaseSequence):
         paths: Sequence[str],
         num_samples: int,
         *,
-        bounds: dict[str, Sequence[float]] | None = None,
+        bounds: dict[str, dict[str, str | Sequence[float]]] | None = None,
         scramble: bool = True,
         strength: int = 1,
         optimization: Literal["random-cd", "lloyd"] | None = None,
@@ -1634,6 +1650,32 @@ class MultiPathSequence(_BaseSequence):
     ) -> Self:
         """
         Generate values from SciPy's latin hypercube sampler: :class:`scipy.stats.qmc.LatinHypercube`.
+
+        Parameters
+        ----------
+        paths : Sequence[str]
+            List of dot-delimited paths within the parameter's nested data structure for which
+            'value' should be set.
+        num_samples : int
+            Number of random hypercube samples to take.
+        bounds : dict[str, dict[str, str  |  Sequence[float]]] | None, optional
+            Bounds dictionary structure which takes a path as a key and returns another dictionary
+            which takes `scaling` and `extent` as keys. `extent` defines the width of the parameter
+            space, and `scaling` defines whether to take logarithmically spaced samples ("log") or not ("linear"). By default,
+            linear scaling and an extent between 0 and 1 is used.
+        scramble : bool, optional
+            See `scipy.stats.qmc.LatinHypercube`, by default True
+        strength : int, optional
+            See 'scipy.stats.qmc.LatinHypercube', by default 1
+        optimization : Literal["random, optional
+            See 'scipy.stats.qmc.LatinHypercube', by default None
+        rng : _type_, optional
+            See 'scipy.stats.qmc.LatinHypercube', by default None
+
+        Returns
+        -------
+        NDArray
+            Array of hypercube samples.
         """
         kwargs = {
             "paths": paths,

{hpcflow_new2-0.2.0a222 → hpcflow_new2-0.2.0a227}/hpcflow/sdk/persistence/base.py

@@ -1218,12 +1218,15 @@ class PersistentStore(
     @contextlib.contextmanager
     def cache_ctx(self) -> Iterator[None]:
         """Context manager for using the persistent element/iteration/run cache."""
-        self._use_cache = True
-        try:
+        if self._use_cache:
             yield
-        finally:
-            self._use_cache = False
-            self._reset_cache()
+        else:
+            self._use_cache = True
+            try:
+                yield
+            finally:
+                self._use_cache = False
+                self._reset_cache()
 
     @contextlib.contextmanager
     def parameters_metadata_cache(self):

{hpcflow_new2-0.2.0a222 → hpcflow_new2-0.2.0a227}/hpcflow/sdk/persistence/zarr.py

@@ -18,6 +18,7 @@ from numpy.ma.core import MaskedArray
 import zarr  # type: ignore
 from zarr.errors import BoundsCheckError  # type: ignore
 from zarr.storage import DirectoryStore, FSStore  # type: ignore
+from zarr.util import guess_chunks  # type: ignore
 from fsspec.implementations.zip import ZipFileSystem  # type: ignore
 from rich.console import Console
 from numcodecs import MsgPack, VLenArray, blosc, Blosc, Zstd  # type: ignore
@@ -59,6 +60,7 @@ from hpcflow.sdk.submission.submission import (
     SUBMISSION_SUBMIT_TIME_KEYS,
 )
 from hpcflow.sdk.utils.arrays import get_2D_idx, split_arr
+from hpcflow.sdk.utils.patches import override_module_attrs
 from hpcflow.sdk.utils.strings import shorten_list_str
 
 if TYPE_CHECKING:
@@ -89,6 +91,11 @@ if TYPE_CHECKING:
 ListAny: TypeAlias = "list[Any]"
 #: Zarr attribute mapping context.
 ZarrAttrs: TypeAlias = "dict[str, Any]"
+#: Soft lower limit for the number of bytes in an array chunk
+_ARRAY_CHUNK_MIN: int = 500 * 1024 * 1024  # 500 MiB
+#: Hard upper limit for the number of bytes in an array chunk. Should be lower than the
+#: maximum buffer size of the blosc encoder, if we're using it (2 GiB)
+_ARRAY_CHUNK_MAX: int = 1024 * 1024 * 1024  # 1 GiB
 _JS: TypeAlias = "dict[str, list[dict[str, dict]]]"
 
 
@@ -124,7 +131,13 @@ def _encode_numpy_array(
     new_idx = (
         max((int(i.removeprefix("arr_")) for i in param_arr_group.keys()), default=-1) + 1
     )
-    param_arr_group.create_dataset(name=f"arr_{new_idx}", data=obj, chunks=obj.shape)
+    with override_module_attrs(
+        "zarr.util", {"CHUNK_MIN": _ARRAY_CHUNK_MIN, "CHUNK_MAX": _ARRAY_CHUNK_MAX}
+    ):
+        # `guess_chunks` also ensures chunk shape is at least 1 in each dimension:
+        chunk_shape = guess_chunks(obj.shape, obj.dtype.itemsize)
+
+    param_arr_group.create_dataset(name=f"arr_{new_idx}", data=obj, chunks=chunk_shape)
     type_lookup["arrays"].append([path, new_idx])
 
     return len(type_lookup["arrays"]) - 1

{hpcflow_new2-0.2.0a222 → hpcflow_new2-0.2.0a227}/hpcflow/sdk/submission/enums.py

@@ -6,82 +6,56 @@ from __future__ import annotations
 from dataclasses import dataclass
 from enum import Enum
 
+from hpcflow.sdk.core.enums import _ReportableStateData, _ReportableStateEnum
 
-@dataclass(frozen=True)
-class _JES:
-    """
-    Model of the state of a JobscriptElementState
-    """
-
-    _value: int
-    #: The symbol used to render the state.
-    symbol: str
-    #: The colour used to render the state.
-    colour: str
-    __doc__: str = ""
 
-
-class JobscriptElementState(_JES, Enum):
+class JobscriptElementState(_ReportableStateEnum):
     """Enumeration to convey a particular jobscript element state as reported by the
     scheduler."""
 
     #: Waiting for resource allocation.
-    pending = (
+    pending = _ReportableStateData(
         0,
         "○",
         "yellow",
         "Waiting for resource allocation.",
     )
     #: Waiting for one or more dependencies to finish.
-    waiting = (
+    waiting = _ReportableStateData(
         1,
         "◊",
         "grey46",
         "Waiting for one or more dependencies to finish.",
     )
     #: Executing now.
-    running = (
+    running = _ReportableStateData(
         2,
         "●",
         "dodger_blue1",
         "Executing now.",
     )
     #: Previously submitted but is no longer active.
-    finished = (
+    finished = _ReportableStateData(
         3,
         "■",
         "grey46",
         "Previously submitted but is no longer active.",
     )
     #: Cancelled by the user.
-    cancelled = (
+    cancelled = _ReportableStateData(
         4,
         "C",
         "red3",
         "Cancelled by the user.",
     )
     #: The scheduler reports an error state.
-    errored = (
+    errored = _ReportableStateData(
         5,
         "E",
         "red3",
         "The scheduler reports an error state.",
     )
 
-    @property
-    def value(self) -> int:
-        """
-        The numerical value of this state.
-        """
-        return self._value
-
-    @property
-    def rich_repr(self) -> str:
-        """
-        Rich representation of this enumeration element.
-        """
-        return f"[{self.colour}]{self.symbol}[/{self.colour}]"
-
 
 class SubmissionStatus(Enum):
     """

{hpcflow_new2-0.2.0a222 → hpcflow_new2-0.2.0a227}/hpcflow/sdk/submission/jobscript.py

@@ -901,6 +901,7 @@ class Jobscript(JSONLike):
         )
 
     @property
+    @TimeIt.decorator
     def all_EAR_IDs(self) -> NDArray:
         """Return all run IDs of this jobscripts (across all blocks), removing missing
         run IDs (i.e. -1 values)"""
@@ -912,6 +913,8 @@ class Jobscript(JSONLike):
         """
         Description of EAR information for this jobscript.
         """
+        if self.submission._use_EARs_cache:
+            return [self.submission._EARs_cache[ear_id] for ear_id in self.all_EAR_IDs]
         return self.workflow.get_EARs_from_IDs(self.all_EAR_IDs)
 
     @property

{hpcflow_new2-0.2.0a222 → hpcflow_new2-0.2.0a227}/hpcflow/sdk/submission/submission.py

@@ -11,6 +11,7 @@ from textwrap import indent
 from typing import Any, Literal, overload, TYPE_CHECKING
 from typing_extensions import override
 import warnings
+from contextlib import contextmanager
 
 
 from hpcflow.sdk.utils.strings import shorten_list_str
@@ -135,6 +136,10 @@ class Submission(JSONLike):
             None  # assigned on first access
         )
 
+        # updated in _submission_EARs_cache context manager:
+        self._use_EARs_cache = False
+        self._EARs_cache: dict[int, ElementActionRun] = {}
+
         if workflow:
             #: The workflow this is part of.
             self.workflow = workflow
@@ -226,6 +231,8 @@ class Submission(JSONLike):
         del dct["_workflow"]
         del dct["_index"]
         del dct["_submission_parts_lst"]
+        del dct["_use_EARs_cache"]
+        del dct["_EARs_cache"]
         return {k.lstrip("_"): v for k, v in dct.items()}
 
     @property
@@ -265,6 +272,30 @@ class Submission(JSONLike):
             ]
         return self._submission_parts_lst
 
+    @property
+    @TimeIt.decorator
+    def use_EARs_cache(self) -> bool:
+        """Whether to pre-cache all EARs associated with the submission."""
+        return self._use_EARs_cache
+
+    @use_EARs_cache.setter
+    @TimeIt.decorator
+    def use_EARs_cache(self, value: bool):
+        """Toggle the EAR caching facility."""
+        if self._use_EARs_cache == value:
+            return
+        self._use_EARs_cache = value
+        if value:
+            all_EAR_IDs = list(self.all_EAR_IDs)
+            self._EARs_cache = {
+                ear_ID: ear
+                for ear_ID, ear in zip(
+                    all_EAR_IDs, self.workflow.get_EARs_from_IDs(all_EAR_IDs)
+                )
+            }
+        else:
+            self._EARs_cache = {}  # reset the cache
+
     @TimeIt.decorator
     def get_start_time(self, submit_time: str) -> datetime | None:
         """Get the start time of a given submission part."""
@@ -283,17 +314,36 @@ class Submission(JSONLike):
     @TimeIt.decorator
     def start_time(self) -> datetime | None:
         """Get the first non-None start time over all submission parts."""
-        times = (
-            self.get_start_time(submit_time) for submit_time in self._submission_parts
-        )
-        return min((t for t in times if t is not None), default=None)
+        with self.using_EARs_cache():
+            times = (
+                self.get_start_time(submit_time) for submit_time in self._submission_parts
+            )
+            return min((t for t in times if t is not None), default=None)
 
     @property
     @TimeIt.decorator
     def end_time(self) -> datetime | None:
         """Get the final non-None end time over all submission parts."""
-        times = (self.get_end_time(submit_time) for submit_time in self._submission_parts)
-        return max((t for t in times if t is not None), default=None)
+        with self.using_EARs_cache():
+            times = (
+                self.get_end_time(submit_time) for submit_time in self._submission_parts
+            )
+            return max((t for t in times if t is not None), default=None)
+
+    @contextmanager
+    def using_EARs_cache(self):
+        """
+        A context manager to load and cache all EARs associated with this submission (and
+        its jobscripts).
+        """
+        if self.use_EARs_cache:
+            yield
+        else:
+            self.use_EARs_cache = True
+            try:
+                yield
+            finally:
+                self.use_EARs_cache = False
 
     @property
     def jobscripts(self) -> list[Jobscript]:
@@ -592,15 +642,18 @@ class Submission(JSONLike):
         """
         The IDs of all EARs in this submission.
         """
-        return (i for js in self.jobscripts for i in js.all_EAR_IDs)
+        return (int(i) for js in self.jobscripts for i in js.all_EAR_IDs)
 
     @property
     @TimeIt.decorator
-    def all_EARs(self) -> Iterable[ElementActionRun]:
+    def all_EARs(self) -> list[ElementActionRun]:
         """
         All EARs in this submission.
         """
-        return (ear for js in self.jobscripts for ear in js.all_EARs)
+        if self.use_EARs_cache:
+            return list(self._EARs_cache.values())
+        else:
+            return self.workflow.get_EARs_from_IDs(self.all_EAR_IDs)
 
     @property
     @TimeIt.decorator
@@ -610,9 +663,10 @@ class Submission(JSONLike):
     @property
     @TimeIt.decorator
     def all_EARs_by_jobscript(self) -> list[list[ElementActionRun]]:
-        ids = [i.all_EAR_IDs for i in self.jobscripts]
-        all_EARs = {i.id_: i for i in self.workflow.get_EARs_from_IDs(self.all_EAR_IDs)}
-        return [[all_EARs[i] for i in js_ids] for js_ids in ids]
+        all_EARs = {i.id_: i for i in self.all_EARs}
+        return [
+            [all_EARs[i] for i in js_ids] for js_ids in self.all_EARs_IDs_by_jobscript
+        ]
 
     @property
     @TimeIt.decorator
@@ -650,11 +704,12 @@ class Submission(JSONLike):
         """Get jobscripts that are active on this machine, and their active states."""
         # this returns: {JS_IDX: {BLOCK_IDX: {JS_ELEMENT_IDX: STATE}}}
         # TODO: query the scheduler once for all jobscripts?
-        return {
-            js.index: act_states
-            for js in self.jobscripts
-            if (act_states := js.get_active_states(as_json=as_json))
-        }
+        with self.using_EARs_cache():
+            return {
+                js.index: act_states
+                for js in self.jobscripts
+                if (act_states := js.get_active_states(as_json=as_json))
+            }
 
     @TimeIt.decorator
     def _write_scripts(

hpcflow_new2-0.2.0a227/hpcflow/sdk/utils/patches.py

@@ -0,0 +1,31 @@
+from contextlib import contextmanager
+from pathlib import Path
+import sys
+from typing import Any
+
+
+def resolve_path(path):
+    """On Windows Python 3.8, 3.9, and 3.10, `Pathlib.resolve` does
+    not return an absolute path for non-existant paths, when it should.
+
+    See: https://github.com/python/cpython/issues/82852
+
+    """
+    # TODO: this only seems to be used in a test; remove?
+    return Path.cwd() / Path(path).resolve()  # cwd is ignored if already absolute
+
+
+@contextmanager
+def override_module_attrs(module_name: str, overrides: dict[str, Any]):
+    """Context manager to temporarily override module-level attributes. The module must be
+    imported (i.e. within `sys.modules`)."""
+
+    module = sys.modules[module_name]
+    original_values = {k: getattr(module, k) for k in overrides}
+    try:
+        for k, v in overrides.items():
+            setattr(module, k, v)
+        yield
+    finally:
+        for k, v in original_values.items():
+            setattr(module, k, v)