hpcflow-new2 0.2.0a222__py3-none-any.whl → 0.2.0a227__py3-none-any.whl

hpcflow/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0a222"
+__version__ = "0.2.0a227"

hpcflow/sdk/app.py

@@ -3315,12 +3315,13 @@ class BaseApp(metaclass=Singleton):
                     out_item["deleted"] = True
 
                 else:
-                    if status:
-                        status.update(
-                            f"Reading workflow {file_dat_i['path']!r} submission info..."
-                        )
                     with wk_i._store.cache_ctx():
                         sub = wk_i.submissions[file_dat_i["sub_idx"]]
+                        if status:
+                            status.update(
+                                f"Loading workflow {file_dat_i['path']!r} run metadata..."
+                            )
+                        sub.use_EARs_cache = True  # pre-cache EARs of this submission
 
                         all_jobscripts = sub._submission_parts[submit_time_str]
                         out_item["jobscripts"] = all_jobscripts
@@ -3583,7 +3584,7 @@ class BaseApp(metaclass=Singleton):
                         )
                         all_cells["status"] = "/".join(
                             js_state.rich_repr
-                            for js_state in sorted(act_js_states, key=lambda x: x.value)
+                            for js_state in sorted(act_js_states, key=lambda x: x.id)
                         )
                     else:
                         if deleted:
@@ -3645,7 +3646,7 @@ class BaseApp(metaclass=Singleton):
                         all_cells["actions_compact"] = " | ".join(
                             f"[{k.colour}]{k.symbol}[/{k.colour}]:{v}"  # type: ignore
                             for k, v in dict(
-                                sorted(EAR_stat_count.items(), key=lambda x: x[0].value)
+                                sorted(EAR_stat_count.items(), key=lambda x: x[0].id)
                             ).items()
                         )
                     else:

hpcflow/sdk/cli.py

@@ -4,6 +4,7 @@ Command line interface implementation.
 
 from __future__ import annotations
 import contextlib
+import datetime
 import json
 import os
 import time
@@ -772,6 +773,12 @@ def _make_submission_CLI(app: BaseApp):
     def get_login_nodes(scheduler: SGEPosix):
         pprint(scheduler.get_login_nodes())
 
+    class _DateTimeJSONEncoder(json.JSONEncoder):
+        def default(self, obj):
+            if isinstance(obj, datetime.datetime):
+                return obj.isoformat()
+            return super().default(obj)
+
     @submission.command()
     @click.option(
         "as_json",
@@ -784,7 +791,7 @@ def _make_submission_CLI(app: BaseApp):
         """Print known-submissions information as a formatted Python object."""
         out = app.get_known_submissions(as_json=as_json)
         if as_json:
-            click.echo(json.dumps(out))
+            click.echo(json.dumps(out, cls=_DateTimeJSONEncoder))
         else:
             pprint(out)
 

hpcflow/sdk/core/enums.py

@@ -26,45 +26,85 @@ class ActionScopeType(Enum):
 
 
 @dataclass(frozen=True)
-class _EARStatus:
+class _ReportableStateData:
     """
-    Model of the state of an EARStatus.
+    Model of the state of something that is renderable using a symbol and colour.
+
+    Notes
+    -----
+    This class is used as the value in the the enumeration subclasses
+    :py:class:`EARStatus` and :py:class:`JobscriptElementState`.
+
     """
 
-    _value: int
-    #: Symbol to use when rendering a status.
+    #: ID that distinguishes the state.
+    id: int
+    #: Symbol to use when rendering a state.
     symbol: str
-    #: Colour to use when rendering a status.
+    #: Colour to use when rendering a state.
     colour: str
+    #: Documentation of the meaning of the state.
     __doc__: str = ""
 
 
-class EARStatus(_EARStatus, Enum):
+class _ReportableStateEnum(Enum):
+    """Enumeration superclass for reportable state subclasses with some shared methods."""
+
+    @property
+    def id(self) -> int:
+        """
+        The integer ID associated with this state.
+        """
+        return self.value.id
+
+    @property
+    def colour(self) -> str:
+        """
+        The colour associated with this state.
+        """
+        return self.value.colour
+
+    @property
+    def symbol(self) -> str:
+        """
+        The symbol associated with this state.
+        """
+        return self.value.symbol
+
+    @property
+    def rich_repr(self) -> str:
+        """
+        Rich representation of this enumeration element.
+        """
+        return f"[{self.colour}]{self.symbol}[/{self.colour}]"
+
+
+class EARStatus(_ReportableStateEnum):
     """Enumeration of all possible EAR statuses, and their associated status colour."""
 
     #: Not yet associated with a submission.
-    pending = (
+    pending = _ReportableStateData(
         0,
         ".",
         "grey46",
         "Not yet associated with a submission.",
     )
     #: Associated with a prepared submission that is not yet submitted.
-    prepared = (
+    prepared = _ReportableStateData(
         1,
         ".",
         "grey46",
         "Associated with a prepared submission that is not yet submitted.",
     )
     #: Submitted for execution.
-    submitted = (
+    submitted = _ReportableStateData(
         2,
         ".",
         "grey46",
         "Submitted for execution.",
     )
     #: Executing now.
-    running = (
+    running = _ReportableStateData(
         3,
         "●",
         "dodger_blue1",
@@ -72,7 +112,7 @@ class EARStatus(_EARStatus, Enum):
     )
     #: Not attempted due to a failure of an upstream action on which this depends,
     #: or a loop termination condition being satisfied.
-    skipped = (
+    skipped = _ReportableStateData(
         4,
         "s",
         "dark_orange",
@@ -82,32 +122,27 @@ class EARStatus(_EARStatus, Enum):
         ),
     )
     #: Aborted by the user; downstream actions will be attempted.
-    aborted = (
+    aborted = _ReportableStateData(
         5,
         "A",
         "deep_pink4",
         "Aborted by the user; downstream actions will be attempted.",
     )
     #: Probably exited successfully.
-    success = (
+    success = _ReportableStateData(
         6,
         "■",
         "green3",
         "Probably exited successfully.",
     )
     #: Probably failed.
-    error = (
+    error = _ReportableStateData(
         7,
         "E",
         "red3",
         "Probably failed.",
     )
 
-    @property
-    def value(self) -> int:
-        #: The value of the status.
-        return self._value
-
     @classmethod
     def get_non_running_submitted_states(cls) -> frozenset[EARStatus]:
         """Return the set of all non-running states, excluding those before submission."""
@@ -120,13 +155,6 @@ class EARStatus(_EARStatus, Enum):
             }
         )
 
-    @property
-    def rich_repr(self) -> str:
-        """
-        The rich representation of the value.
-        """
-        return f"[{self.colour}]{self.symbol}[/{self.colour}]"
-
 
 class InputSourceType(Enum):
     """

hpcflow/sdk/core/parameters.py

@@ -1584,7 +1584,7 @@ class MultiPathSequence(_BaseSequence):
         paths: Sequence[str],
         num_samples: int,
         *,
-        bounds: dict[str, Sequence[float]] | None = None,
+        bounds: dict[str, dict[str, str | Sequence[float]]] | None = None,
         scramble: bool = True,
         strength: int = 1,
         optimization: Literal["random-cd", "lloyd"] | None = None,
@@ -1602,10 +1602,20 @@ class MultiPathSequence(_BaseSequence):
 
         bounds = bounds or {}
 
-        parameter_ranges = np.array([bounds.get(path, [0, 1]) for path in paths]).T
+        scaling = np.asarray(
+            [bounds.get(path, {}).get("scaling", "linear") for path in paths]
+        )
+
+        # extents including defaults for unspecified:
+        all_extents = [bounds.get(path, {}).get("extent", [0, 1]) for path in paths]
 
-        lower_bound = parameter_ranges[0]
-        upper_bound = parameter_ranges[1]
+        # extents accounting for scaling type:
+        extent = np.asarray(
+            [
+                np.log10(all_extents[i]) if scaling[i] == "log" else all_extents[i]
+                for i in range(len(scaling))
+            ]
+        ).T
 
         try:
             sampler = LatinHypercube(**kwargs)
@@ -1614,9 +1624,15 @@ class MultiPathSequence(_BaseSequence):
             kwargs["seed"] = kwargs.pop("rng")
             sampler = LatinHypercube(**kwargs)
 
-        samples = scale(sampler.random(n=num_samples), lower_bound, upper_bound).T
+        samples = scale(
+            sampler.random(n=num_samples), l_bounds=extent[0], u_bounds=extent[1]
+        )
+
+        for i in range(len(scaling)):
+            if scaling[i] == "log":
+                samples[:, i] = 10 ** samples[:, i]
 
-        return samples
+        return samples.T
 
     @classmethod
     def from_latin_hypercube(
@@ -1624,7 +1640,7 @@ class MultiPathSequence(_BaseSequence):
         paths: Sequence[str],
         num_samples: int,
         *,
-        bounds: dict[str, Sequence[float]] | None = None,
+        bounds: dict[str, dict[str, str | Sequence[float]]] | None = None,
         scramble: bool = True,
         strength: int = 1,
         optimization: Literal["random-cd", "lloyd"] | None = None,
@@ -1634,6 +1650,32 @@ class MultiPathSequence(_BaseSequence):
     ) -> Self:
         """
         Generate values from SciPy's latin hypercube sampler: :class:`scipy.stats.qmc.LatinHypercube`.
+
+        Parameters
+        ----------
+        paths : Sequence[str]
+            List of dot-delimited paths within the parameter's nested data structure for which
+            'value' should be set.
+        num_samples : int
+            Number of random hypercube samples to take.
+        bounds : dict[str, dict[str, str  |  Sequence[float]]] | None, optional
+            Bounds dictionary structure which takes a path as a key and returns another dictionary
+            which takes `scaling` and `extent` as keys. `extent` defines the width of the parameter
+            space, and `scaling` defines whether to take logarithmically spaced samples ("log") or not ("linear"). By default,
+            linear scaling and an extent between 0 and 1 is used.
+        scramble : bool, optional
+            See `scipy.stats.qmc.LatinHypercube`, by default True
+        strength : int, optional
+            See 'scipy.stats.qmc.LatinHypercube', by default 1
+        optimization : Literal["random, optional
+            See 'scipy.stats.qmc.LatinHypercube', by default None
+        rng : _type_, optional
+            See 'scipy.stats.qmc.LatinHypercube', by default None
+
+        Returns
+        -------
+        NDArray
+            Array of hypercube samples.
         """
         kwargs = {
             "paths": paths,

hpcflow/sdk/persistence/base.py

@@ -1218,12 +1218,15 @@ class PersistentStore(
     @contextlib.contextmanager
     def cache_ctx(self) -> Iterator[None]:
         """Context manager for using the persistent element/iteration/run cache."""
-        self._use_cache = True
-        try:
+        if self._use_cache:
             yield
-        finally:
-            self._use_cache = False
-            self._reset_cache()
+        else:
+            self._use_cache = True
+            try:
+                yield
+            finally:
+                self._use_cache = False
+                self._reset_cache()
 
     @contextlib.contextmanager
     def parameters_metadata_cache(self):

hpcflow/sdk/persistence/zarr.py

@@ -18,6 +18,7 @@ from numpy.ma.core import MaskedArray
 import zarr  # type: ignore
 from zarr.errors import BoundsCheckError  # type: ignore
 from zarr.storage import DirectoryStore, FSStore  # type: ignore
+from zarr.util import guess_chunks  # type: ignore
 from fsspec.implementations.zip import ZipFileSystem  # type: ignore
 from rich.console import Console
 from numcodecs import MsgPack, VLenArray, blosc, Blosc, Zstd  # type: ignore
@@ -59,6 +60,7 @@ from hpcflow.sdk.submission.submission import (
     SUBMISSION_SUBMIT_TIME_KEYS,
 )
 from hpcflow.sdk.utils.arrays import get_2D_idx, split_arr
+from hpcflow.sdk.utils.patches import override_module_attrs
 from hpcflow.sdk.utils.strings import shorten_list_str
 
 if TYPE_CHECKING:
@@ -89,6 +91,11 @@ if TYPE_CHECKING:
 ListAny: TypeAlias = "list[Any]"
 #: Zarr attribute mapping context.
 ZarrAttrs: TypeAlias = "dict[str, Any]"
+#: Soft lower limit for the number of bytes in an array chunk
+_ARRAY_CHUNK_MIN: int = 500 * 1024 * 1024  # 500 MiB
+#: Hard upper limit for the number of bytes in an array chunk. Should be lower than the
+#: maximum buffer size of the blosc encoder, if we're using it (2 GiB)
+_ARRAY_CHUNK_MAX: int = 1024 * 1024 * 1024  # 1 GiB
 _JS: TypeAlias = "dict[str, list[dict[str, dict]]]"
 
 
@@ -124,7 +131,13 @@ def _encode_numpy_array(
     new_idx = (
         max((int(i.removeprefix("arr_")) for i in param_arr_group.keys()), default=-1) + 1
     )
-    param_arr_group.create_dataset(name=f"arr_{new_idx}", data=obj, chunks=obj.shape)
+    with override_module_attrs(
+        "zarr.util", {"CHUNK_MIN": _ARRAY_CHUNK_MIN, "CHUNK_MAX": _ARRAY_CHUNK_MAX}
+    ):
+        # `guess_chunks` also ensures chunk shape is at least 1 in each dimension:
+        chunk_shape = guess_chunks(obj.shape, obj.dtype.itemsize)
+
+    param_arr_group.create_dataset(name=f"arr_{new_idx}", data=obj, chunks=chunk_shape)
     type_lookup["arrays"].append([path, new_idx])
 
     return len(type_lookup["arrays"]) - 1

hpcflow/sdk/submission/enums.py

@@ -6,82 +6,56 @@ from __future__ import annotations
 from dataclasses import dataclass
 from enum import Enum
 
+from hpcflow.sdk.core.enums import _ReportableStateData, _ReportableStateEnum
 
-@dataclass(frozen=True)
-class _JES:
-    """
-    Model of the state of a JobscriptElementState
-    """
-
-    _value: int
-    #: The symbol used to render the state.
-    symbol: str
-    #: The colour used to render the state.
-    colour: str
-    __doc__: str = ""
 
-
-class JobscriptElementState(_JES, Enum):
+class JobscriptElementState(_ReportableStateEnum):
     """Enumeration to convey a particular jobscript element state as reported by the
     scheduler."""
 
     #: Waiting for resource allocation.
-    pending = (
+    pending = _ReportableStateData(
         0,
         "○",
         "yellow",
         "Waiting for resource allocation.",
     )
     #: Waiting for one or more dependencies to finish.
-    waiting = (
+    waiting = _ReportableStateData(
         1,
         "◊",
         "grey46",
         "Waiting for one or more dependencies to finish.",
     )
     #: Executing now.
-    running = (
+    running = _ReportableStateData(
         2,
         "●",
         "dodger_blue1",
         "Executing now.",
     )
     #: Previously submitted but is no longer active.
-    finished = (
+    finished = _ReportableStateData(
         3,
         "■",
         "grey46",
         "Previously submitted but is no longer active.",
     )
     #: Cancelled by the user.
-    cancelled = (
+    cancelled = _ReportableStateData(
         4,
         "C",
         "red3",
         "Cancelled by the user.",
     )
     #: The scheduler reports an error state.
-    errored = (
+    errored = _ReportableStateData(
         5,
         "E",
         "red3",
         "The scheduler reports an error state.",
     )
 
-    @property
-    def value(self) -> int:
-        """
-        The numerical value of this state.
-        """
-        return self._value
-
-    @property
-    def rich_repr(self) -> str:
-        """
-        Rich representation of this enumeration element.
-        """
-        return f"[{self.colour}]{self.symbol}[/{self.colour}]"
-
 
 class SubmissionStatus(Enum):
     """

hpcflow/sdk/submission/jobscript.py

@@ -901,6 +901,7 @@ class Jobscript(JSONLike):
         )
 
     @property
+    @TimeIt.decorator
     def all_EAR_IDs(self) -> NDArray:
         """Return all run IDs of this jobscripts (across all blocks), removing missing
         run IDs (i.e. -1 values)"""
@@ -912,6 +913,8 @@ class Jobscript(JSONLike):
         """
         Description of EAR information for this jobscript.
         """
+        if self.submission._use_EARs_cache:
+            return [self.submission._EARs_cache[ear_id] for ear_id in self.all_EAR_IDs]
         return self.workflow.get_EARs_from_IDs(self.all_EAR_IDs)
 
     @property

hpcflow/sdk/submission/submission.py

@@ -11,6 +11,7 @@ from textwrap import indent
 from typing import Any, Literal, overload, TYPE_CHECKING
 from typing_extensions import override
 import warnings
+from contextlib import contextmanager
 
 
 from hpcflow.sdk.utils.strings import shorten_list_str
@@ -135,6 +136,10 @@ class Submission(JSONLike):
             None  # assigned on first access
         )
 
+        # updated in _submission_EARs_cache context manager:
+        self._use_EARs_cache = False
+        self._EARs_cache: dict[int, ElementActionRun] = {}
+
         if workflow:
             #: The workflow this is part of.
             self.workflow = workflow
@@ -226,6 +231,8 @@ class Submission(JSONLike):
         del dct["_workflow"]
         del dct["_index"]
         del dct["_submission_parts_lst"]
+        del dct["_use_EARs_cache"]
+        del dct["_EARs_cache"]
         return {k.lstrip("_"): v for k, v in dct.items()}
 
     @property
@@ -265,6 +272,30 @@ class Submission(JSONLike):
             ]
         return self._submission_parts_lst
 
+    @property
+    @TimeIt.decorator
+    def use_EARs_cache(self) -> bool:
+        """Whether to pre-cache all EARs associated with the submission."""
+        return self._use_EARs_cache
+
+    @use_EARs_cache.setter
+    @TimeIt.decorator
+    def use_EARs_cache(self, value: bool):
+        """Toggle the EAR caching facility."""
+        if self._use_EARs_cache == value:
+            return
+        self._use_EARs_cache = value
+        if value:
+            all_EAR_IDs = list(self.all_EAR_IDs)
+            self._EARs_cache = {
+                ear_ID: ear
+                for ear_ID, ear in zip(
+                    all_EAR_IDs, self.workflow.get_EARs_from_IDs(all_EAR_IDs)
+                )
+            }
+        else:
+            self._EARs_cache = {}  # reset the cache
+
     @TimeIt.decorator
     def get_start_time(self, submit_time: str) -> datetime | None:
         """Get the start time of a given submission part."""
@@ -283,17 +314,36 @@ class Submission(JSONLike):
     @TimeIt.decorator
     def start_time(self) -> datetime | None:
         """Get the first non-None start time over all submission parts."""
-        times = (
-            self.get_start_time(submit_time) for submit_time in self._submission_parts
-        )
-        return min((t for t in times if t is not None), default=None)
+        with self.using_EARs_cache():
+            times = (
+                self.get_start_time(submit_time) for submit_time in self._submission_parts
+            )
+            return min((t for t in times if t is not None), default=None)
 
     @property
     @TimeIt.decorator
     def end_time(self) -> datetime | None:
         """Get the final non-None end time over all submission parts."""
-        times = (self.get_end_time(submit_time) for submit_time in self._submission_parts)
-        return max((t for t in times if t is not None), default=None)
+        with self.using_EARs_cache():
+            times = (
+                self.get_end_time(submit_time) for submit_time in self._submission_parts
+            )
+            return max((t for t in times if t is not None), default=None)
+
+    @contextmanager
+    def using_EARs_cache(self):
+        """
+        A context manager to load and cache all EARs associated with this submission (and
+        its jobscripts).
+        """
+        if self.use_EARs_cache:
+            yield
+        else:
+            self.use_EARs_cache = True
+            try:
+                yield
+            finally:
+                self.use_EARs_cache = False
 
     @property
     def jobscripts(self) -> list[Jobscript]:
@@ -592,15 +642,18 @@ class Submission(JSONLike):
         """
         The IDs of all EARs in this submission.
         """
-        return (i for js in self.jobscripts for i in js.all_EAR_IDs)
+        return (int(i) for js in self.jobscripts for i in js.all_EAR_IDs)
 
     @property
     @TimeIt.decorator
-    def all_EARs(self) -> Iterable[ElementActionRun]:
+    def all_EARs(self) -> list[ElementActionRun]:
         """
         All EARs in this submission.
         """
-        return (ear for js in self.jobscripts for ear in js.all_EARs)
+        if self.use_EARs_cache:
+            return list(self._EARs_cache.values())
+        else:
+            return self.workflow.get_EARs_from_IDs(self.all_EAR_IDs)
 
     @property
     @TimeIt.decorator
@@ -610,9 +663,10 @@ class Submission(JSONLike):
     @property
     @TimeIt.decorator
     def all_EARs_by_jobscript(self) -> list[list[ElementActionRun]]:
-        ids = [i.all_EAR_IDs for i in self.jobscripts]
-        all_EARs = {i.id_: i for i in self.workflow.get_EARs_from_IDs(self.all_EAR_IDs)}
-        return [[all_EARs[i] for i in js_ids] for js_ids in ids]
+        all_EARs = {i.id_: i for i in self.all_EARs}
+        return [
+            [all_EARs[i] for i in js_ids] for js_ids in self.all_EARs_IDs_by_jobscript
+        ]
 
     @property
     @TimeIt.decorator
@@ -650,11 +704,12 @@ class Submission(JSONLike):
         """Get jobscripts that are active on this machine, and their active states."""
         # this returns: {JS_IDX: {BLOCK_IDX: {JS_ELEMENT_IDX: STATE}}}
         # TODO: query the scheduler once for all jobscripts?
-        return {
-            js.index: act_states
-            for js in self.jobscripts
-            if (act_states := js.get_active_states(as_json=as_json))
-        }
+        with self.using_EARs_cache():
+            return {
+                js.index: act_states
+                for js in self.jobscripts
+                if (act_states := js.get_active_states(as_json=as_json))
+            }
 
     @TimeIt.decorator
     def _write_scripts(

hpcflow/sdk/utils/patches.py

@@ -1,4 +1,7 @@
+from contextlib import contextmanager
 from pathlib import Path
+import sys
+from typing import Any
 
 
 def resolve_path(path):
@@ -10,3 +13,19 @@ def resolve_path(path):
     """
     # TODO: this only seems to be used in a test; remove?
     return Path.cwd() / Path(path).resolve()  # cwd is ignored if already absolute
+
+
+@contextmanager
+def override_module_attrs(module_name: str, overrides: dict[str, Any]):
+    """Context manager to temporarily override module-level attributes. The module must be
+    imported (i.e. within `sys.modules`)."""
+
+    module = sys.modules[module_name]
+    original_values = {k: getattr(module, k) for k in overrides}
+    try:
+        for k, v in overrides.items():
+            setattr(module, k, v)
+        yield
+    finally:
+        for k, v in original_values.items():
+            setattr(module, k, v)

hpcflow/tests/unit/test_multi_path_sequences.py

@@ -172,25 +172,32 @@ def test_MPS_latin_hypercube_sequence_values():
 
 def test_MPS_latin_hypercube_sequence_bounds():
 
-    bounds = {"inputs.a": [16789.2, 17812.5]}
+    bounds = {
+        "inputs.a": {"extent": [16789.2, 17812.5], "scaling": "linear"},
+        "inputs.c": {"extent": [1.0e-10, 1.0e-5], "scaling": "log"},
+    }
 
     mps = hf.MultiPathSequence.from_latin_hypercube(
-        paths=["inputs.a", "inputs.b"],
+        paths=["inputs.a", "inputs.b", "inputs.c"],
         num_samples=10,
         bounds=bounds,
     )
 
     vals_arr = np.array(mps.values)
-    assert vals_arr.shape == (2, 10)
 
-    vals_0 = vals_arr[0]
-    bounds_0 = list(bounds.values())[0]
+    assert vals_arr.shape == (3, 10)
 
-    vals_1 = vals_arr[1]
-    bounds_1 = [0, 1]
+    vals_a = vals_arr[0]
+    vals_b = vals_arr[1]
+    vals_c = vals_arr[2]
 
-    assert np.logical_and(vals_0 > bounds_0[0], vals_0 < bounds_0[1]).all()
-    assert np.logical_and(vals_1 > bounds_1[0], vals_1 < bounds_1[1]).all()
+    extent_a = bounds["inputs.a"]["extent"]
+    extent_b = [0.0, 1.0]
+    extent_c = bounds["inputs.c"]["extent"]
+
+    assert np.logical_and(vals_a > extent_a[0], vals_a < extent_a[1]).all()
+    assert np.logical_and(vals_b > extent_b[0], vals_b < extent_b[1]).all()
+    assert np.logical_and(vals_c > extent_c[0], vals_c < extent_c[1]).all()
 
 
 def test_MPS_move_from_sequences_list():

hpcflow/tests/unit/test_persistence.py

@@ -6,7 +6,11 @@ from typing import cast, TYPE_CHECKING
 import numpy as np
 import zarr  # type: ignore
 import pytest
-from hpcflow.sdk.core.test_utils import make_test_data_YAML_workflow, make_workflow
+from hpcflow.sdk.core.test_utils import (
+    make_schemas,
+    make_test_data_YAML_workflow,
+    make_workflow,
+)
 from hpcflow.sdk.persistence.json import (
     JSONPersistentStore,
     JsonStoreElement,
@@ -551,3 +555,38 @@ def test_zarr_encode_decode_jobscript_block_dependencies_large_one_to_one():
     arr = ZarrPersistentStore._encode_jobscript_block_dependencies(deps_t)
     deps_rt = ZarrPersistentStore._decode_jobscript_block_dependencies(arr)
     assert deps_rt == deps
+
+
+@pytest.mark.parametrize(
+    "array",
+    (
+        np.array([]),
+        np.empty(0),
+        np.empty((0, 1, 2)),
+        np.array([1, 2, 3]),
+        np.array([[1, 2, 3], [4, 5, 6]]),
+    ),
+)
+def test_zarr_save_persistent_array_shape(null_config, tmp_path, array):
+    s1 = make_schemas(({"p1": None}, ()))
+    t1 = hf.Task(schema=s1, inputs={"p1": array})
+    wk = hf.Workflow.from_template_data(
+        template_name="test_save_empty_array",
+        tasks=[t1],
+        path=tmp_path,
+    )
+    assert array.shape == wk.tasks[0].elements[0].get("inputs.p1")[:].shape
+
+
+def test_zarr_single_chunk_threshold(null_config, tmp_path):
+    # test very large arrays (> ~1 GB) are saved using multiple chunks
+    array = np.zeros(
+        268_435_456
+    )  # ~ 2.147483647 GB; greater than blosc's max buffer size
+    s1 = make_schemas(({"p1": None}, ()))
+    t1 = hf.Task(schema=s1, inputs={"p1": array})
+    wk = hf.Workflow.from_template_data(
+        template_name="test_large_array",
+        tasks=[t1],
+        path=tmp_path,
+    )

{hpcflow_new2-0.2.0a222.dist-info → hpcflow_new2-0.2.0a227.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a222
+Version: 0.2.0a227
 Summary: Computational workflow management
 License: MPL-2.0
 Author: aplowman

{hpcflow_new2-0.2.0a222.dist-info → hpcflow_new2-0.2.0a227.dist-info}/RECORD

@@ -1,7 +1,7 @@
 hpcflow/__init__.py,sha256=WIETuRHeOp2SqUqHUzpjQ-lk9acbYv-6aWOhZPRdlhs,64
 hpcflow/__pyinstaller/__init__.py,sha256=YOzBlPSck6slucv6lJM9K80JtsJWxXRL00cv6tRj3oc,98
 hpcflow/__pyinstaller/hook-hpcflow.py,sha256=P2b-8QdQqkSS7cJB6CB3CudUuJ9iZzTh2fQF4hNdCa4,1118
-hpcflow/_version.py,sha256=phew7JkOXGWTg0xqEm-B1aQnoPYpQVwtKpvMwtWxwkU,26
+hpcflow/_version.py,sha256=Jgo3POaN0i9GWVRAfP7uEOtWUMY-0Fis95oTeRaSmHQ,26
 hpcflow/app.py,sha256=gl2viVS65PbpDhUp2DARaYHFDqDWQjuoyB3ikrCNRW4,1367
 hpcflow/cli.py,sha256=G2J3D9v6MnMWOWMMWK6UEKLn_6wnV9lT_qygEBBxg-I,66
 hpcflow/data/demo_data_manifest/__init__.py,sha256=Hsq0jT8EXM13wu1MpGy5FQgyuz56ygep4VWOnulFn50,41
@@ -64,8 +64,8 @@ hpcflow/data/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 hpcflow/data/workflows/workflow_1.yaml,sha256=lF7Re2SVc_5gQk5AwB0gXaq-n-T5ia4su3zNQ9oMRV0,220
 hpcflow/examples.ipynb,sha256=cLKp4QsxwwMXRanDnfWY9kqsV23q6G4raOpu6IZXnMw,28553
 hpcflow/sdk/__init__.py,sha256=BEbIz8adCAX06ZmvEcquEu29PmQDvVP1I1qn_q78dYc,6289
-hpcflow/sdk/app.py,sha256=ZFMnEV43dlZlnTpJNzBsfRZ1ndiMGyeJ3hKxlhrm39I,146726
-hpcflow/sdk/cli.py,sha256=C_7Ts6TbspTZtUXlPwF37Vt-t71NABJ_2zNpZLYFPUE,45563
+hpcflow/sdk/app.py,sha256=UWnpxdEqVD07VE-Rr23xhVHesF3zZI6ryLJ1-yXKp7M,146820
+hpcflow/sdk/cli.py,sha256=qMFACsIA32gD2oLC8_FD9JjFYgidCoGxOH-L3qM2xx0,45818
 hpcflow/sdk/cli_common.py,sha256=MO0TbC280JdZTLnxSV15wE7PDNx83sotpJsyAN1E2F4,10458
 hpcflow/sdk/config/__init__.py,sha256=pkufl_CXm_Kswk1x7LTavuplWjUbUCitK0AJ2E60gwM,137
 hpcflow/sdk/config/callbacks.py,sha256=cuW31ZW0YTOi5K5P-NYHuiK1YLsglMEH3c_u6mddVE8,8078
@@ -81,7 +81,7 @@ hpcflow/sdk/core/cache.py,sha256=6b0emvJGP-yf5UxKmEY_s4_gdyUfaNaXafq-qQaoBfU,817
 hpcflow/sdk/core/command_files.py,sha256=LcrK7wjQnxmTDUrsC0Xgbjsy1NGodf62LrrTSNAAxxE,24942
 hpcflow/sdk/core/commands.py,sha256=vFyZ9Cv_tvGqSGtVEnAHAeXA-HAxWoF0nmf1e0bFCvs,15993
 hpcflow/sdk/core/element.py,sha256=pPRezx0HATKH55qEstXp1xOG4S8oGLVnJ3sQcEmLsXo,68152
-hpcflow/sdk/core/enums.py,sha256=BV_cFqTWkME3zVLswmb21QXeIiCBb9_9avr6VIhL3HM,4387
+hpcflow/sdk/core/enums.py,sha256=scp7b7gbFSefu1bks2M2u6amq6G_lB_j_h8U_TPM4bU,5316
 hpcflow/sdk/core/environment.py,sha256=m0Qp4Y1XcPJ1yPrdHp6RJv9_H61nA9Qo2ZJV_-wGne8,7959
 hpcflow/sdk/core/errors.py,sha256=8PxlFewrRZU2MSPoNUiQ2jAX9iQTr9-Qr-O5hDywU_A,28869
 hpcflow/sdk/core/execute.py,sha256=Gc2SauA4QNA1W4Gi2FYoKoqkoJtGg57cgn1p87WNCIs,7221
@@ -89,7 +89,7 @@ hpcflow/sdk/core/json_like.py,sha256=tzCpaarGCHr_NXA-vLcX1jrDYlWbZuAbUPjR6jREhX8
 hpcflow/sdk/core/loop.py,sha256=vs7pNVe3Io5zNO6Xl_Tqmra9mCJUw8IESLZC4p2QFvo,50084
 hpcflow/sdk/core/loop_cache.py,sha256=C0t9UpvHFdTmw3bViZ2wccjZWtkbSbN9APvcwzBSREg,10801
 hpcflow/sdk/core/object_list.py,sha256=WvUBb-2DA7-7_7oNoPykSsTihVbGB-1cgHh8W78IzKI,29882
-hpcflow/sdk/core/parameters.py,sha256=5iuqTCz9ZxQnhgDElDIRBB32PDSHnqMJ1701v-M5GVI,98597
+hpcflow/sdk/core/parameters.py,sha256=BlRSdPp_fA4jOfBmaIZ45Y-jGouMYz4tnn5J8Pcmv4w,100384
 hpcflow/sdk/core/rule.py,sha256=LiNyKp09a6TRGJvtiB1CcNqgyAIuPhuSW2RVfDWf5FE,6274
 hpcflow/sdk/core/run_dir_files.py,sha256=yh49BSlswqqyr97x5nDT87qu2MlOLMu7Arzl_B-fxy0,2061
 hpcflow/sdk/core/skip_reason.py,sha256=xPm8bd8sbPpiZ3sx9ZeW0jxgRQ0TBokSv0XIJKrEW2Y,113
@@ -117,7 +117,7 @@ hpcflow/sdk/helper/helper.py,sha256=T45BEwhPgTlwvqDNbT4Us4UVmdRKRoszIMt8a9B69kE,
 hpcflow/sdk/helper/watcher.py,sha256=aQyuSnBiPhsp1ijBylwR5mxjIqlwD4rT_zH9DYc83no,5744
 hpcflow/sdk/log.py,sha256=_wqB0qt65nt4oNNV6eyD39bNZyciyH6biyyOgU_MViQ,8285
 hpcflow/sdk/persistence/__init__.py,sha256=1e6nVADj6xEsE7XGXHaIsHZur4IkUJAxLuxBlvHi0CE,408
-hpcflow/sdk/persistence/base.py,sha256=y7L7FFyuTJvy9K_CRFZZH8Cost2seiNz_KKaKZ2XREo,92727
+hpcflow/sdk/persistence/base.py,sha256=WBkVpbFsNiGjhSNTqmlBs3GO_3yMWCGufSbATYaNlbE,92811
 hpcflow/sdk/persistence/defaults.py,sha256=hpZk2R9HFqo7YTGaIlts6CGBRzQLn-JXebWc2ZHLBe4,129
 hpcflow/sdk/persistence/discovery.py,sha256=jEREI651S3FK-Qoz150oIHP72dsuhCFawXRQIy6j_WQ,1320
 hpcflow/sdk/persistence/json.py,sha256=QUa_xwgcspZWXi6zlIusVRnh4TzDa4xU5UCDcCU6BXE,36746
@@ -125,11 +125,11 @@ hpcflow/sdk/persistence/pending.py,sha256=JB42Emk2cN639bPlNtQtoFhe5WXGWDJeFm7aFU
 hpcflow/sdk/persistence/store_resource.py,sha256=P-VZF7gMIsVvgOsHJEplK1Se4jHsaAqbdoKoQo-7_LU,5903
 hpcflow/sdk/persistence/types.py,sha256=c4vd8RW8bg8Meoz1HNXjGT8Za9Hpfl1Sxp4pLAgdV0g,9125
 hpcflow/sdk/persistence/utils.py,sha256=TuQp0BKwiy1-5WTgiMaYXn00bZb6HWHAjs4BBFE9khw,1804
-hpcflow/sdk/persistence/zarr.py,sha256=yCsiNteaok-eM9x5jogeDoCPdnIpx1Yv7LgYm8qoVeA,88267
+hpcflow/sdk/persistence/zarr.py,sha256=ozJSx0W8L5yVRVvF7gM7M8psF5vd_-dNDnXNa-Rh3sI,88979
 hpcflow/sdk/runtime.py,sha256=vPNu4_DXYMnRyJenkVLq8yt6_XG4tCuwX69Oew4OWmo,11269
 hpcflow/sdk/submission/__init__.py,sha256=79xJXVkddsuj3uJz3BV9iOnAV7vCeJvLrhKccOA_dnU,67
-hpcflow/sdk/submission/enums.py,sha256=DykNHISQDmfa0oDggj6iIKugeFXJd8iF9NwqV3MN2oA,2049
-hpcflow/sdk/submission/jobscript.py,sha256=QvvJAwOAYQ6gkNkRAU1Ke4zJnQd6OneMUkAJapmPMGw,90846
+hpcflow/sdk/submission/enums.py,sha256=W6KDSkDBs-DYj5yZeU0Yt42R7LmomHKJTtPh6jPx3aE,1667
+hpcflow/sdk/submission/jobscript.py,sha256=xxrrcYnmmzyt1nZadIPLdO-ByHYibUJgHk5xfxm7onE,91000
 hpcflow/sdk/submission/schedulers/__init__.py,sha256=E-XBhSMflInTrQ5ZBUy8Hhq-JtAuIcCx6IM_7fI1Axw,9291
 hpcflow/sdk/submission/schedulers/direct.py,sha256=b_LS_xB0vu2tRHs-Xd08-16PC-nojRB6aNXwT9qtv0A,7493
 hpcflow/sdk/submission/schedulers/sge.py,sha256=WRohm-1eBEcadcKGg5QPd1DHvGEUV759IVtLbbDx2qU,13927
@@ -140,13 +140,13 @@ hpcflow/sdk/submission/shells/base.py,sha256=orjshO0jBmcpt2frMk41Cyc6CeFz7mjiiYu
 hpcflow/sdk/submission/shells/bash.py,sha256=x-0hBRX1OlpylWcrHlSS72Bxw8us2ZY7m2PbgWml0-c,16351
 hpcflow/sdk/submission/shells/os_version.py,sha256=_RK5A56png1Zk4_rzumy17xZ4xVguuPMS0Mll99SKaA,3521
 hpcflow/sdk/submission/shells/powershell.py,sha256=OFxSEC9Quh-WDFOeYNCQLX_vg-VzO8KdpsYCS9AQfo0,11725
-hpcflow/sdk/submission/submission.py,sha256=nbbSxf6Kkn2omsZLk5ZQ1PiW4mMttxR_EH7KPLOAXCQ,51985
+hpcflow/sdk/submission/submission.py,sha256=_sCV88L6yFE4BWNpnDjMYzabD_i95IiYqPos0kDOj2s,53659
 hpcflow/sdk/submission/types.py,sha256=8xYzjrFbRlTLtZoOwZJZeY3mDI4nCJlLSJ4ekbDoGVI,4220
 hpcflow/sdk/typing.py,sha256=gu2h7a0RJs_r5UtV8AFVh9TNjo8kGe1n6UOxeH3zdgg,5060
 hpcflow/sdk/utils/arrays.py,sha256=7znIaqfl_bGElZ6U_da0vumWPez6ITyNrSDu_AlACDI,2084
 hpcflow/sdk/utils/deferred_file.py,sha256=LYIjTPjAmbOflwUaq_-o-O6DZJCQzRHX8dl_rIWTO5c,1381
 hpcflow/sdk/utils/hashing.py,sha256=40H7yO-oYgrt6RCdlTDDzJV9YIGiQeDx6zGpMJrRKZo,484
-hpcflow/sdk/utils/patches.py,sha256=jpbodcOjhE5ccrHPz5-L-r9yC7s2ymQccIdJi6Zd2Jo,398
+hpcflow/sdk/utils/patches.py,sha256=sVK8TjeeIEBgngm5PobxkKcIiuDORLaOzI_8ua1H3bw,988
 hpcflow/sdk/utils/strings.py,sha256=MGdVeQpEZjo_9_VSMNI2ArAesQpgAx4-RkHL7IAQInQ,1966
 hpcflow/tests/api/test_api.py,sha256=h0HT9W0Jd1pChrXYaBOVwGThaI3slGkloS0sbq2YX88,962
 hpcflow/tests/conftest.py,sha256=xtSqhOxjZYioiAPvrKwf7NFicZoA4BR9Si4J1A8mWHw,4083
@@ -185,10 +185,10 @@ hpcflow/tests/unit/test_jobscript_unit.py,sha256=E_uszw3-3Lf2nBcIUZUngMMIgd0S1X6
 hpcflow/tests/unit/test_json_like.py,sha256=b0_BHW0zfBmc_J848s0ZcYIl4NJkTxNciiAMMRvUCtA,30434
 hpcflow/tests/unit/test_loop.py,sha256=pN2xVcHhDPYOxR0WA9knVP0F3_PQ3qiXu97wmX32Byc,90046
 hpcflow/tests/unit/test_meta_task.py,sha256=FEWLuUrP-6e6BO0CRJe2fQDDOZUjz1jfljL_-V7n5Gs,9592
-hpcflow/tests/unit/test_multi_path_sequences.py,sha256=6cVYD2e7jMt-m4sr3Z6ixnKewvs-s--slP344trKIWM,9079
+hpcflow/tests/unit/test_multi_path_sequences.py,sha256=eodhujCQkN3DWSyrUHxNlcrIHSOOBE9BnGtRyQnEunc,9362
 hpcflow/tests/unit/test_object_list.py,sha256=uWBou5QCyScaM3DLqL_7RCguMdYZKMkeZR0V8Lpterk,3342
 hpcflow/tests/unit/test_parameter.py,sha256=_ImfDhmroXzPKwIuKiehsAw9qQPPHC9FQa8jN7OJqoA,7440
-hpcflow/tests/unit/test_persistence.py,sha256=cwQoa2czgZ0ZDtCRoRSdkfoFMgMQ5qjnef_mhsMWzrc,17472
+hpcflow/tests/unit/test_persistence.py,sha256=bpBiL4YPMosbZk18Ug1JRqYJ69HHJzDYUff_2wDi-5w,18544
 hpcflow/tests/unit/test_resources.py,sha256=48zC3A5YaSu0MlLlGiohowDpVZNgORsikVX8mP5agvI,8011
 hpcflow/tests/unit/test_run.py,sha256=jFRC1-wV5G2ZCEU-KBKYeHVnJdDBUXX6MbxTrB-RJSE,6205
 hpcflow/tests/unit/test_run_directories.py,sha256=VA2xhDRe7agKO7q1Y8kP0101BstkjLiC1RReKrMI6jE,1088
@@ -218,8 +218,8 @@ hpcflow/tests/workflows/test_submission.py,sha256=SUbBUbD8C8LSulrI7aETkzP9RqED48
 hpcflow/tests/workflows/test_workflows.py,sha256=9z3rtXjA5iMOp4C0q4TkD_9kLzwourCY-obpeOtnNt0,18927
 hpcflow/tests/workflows/test_zip.py,sha256=MzEwsIAYV_1A3bD0XRo23zUwUKVzkkmNd8_cil6YdWQ,578
 hpcflow/viz_demo.ipynb,sha256=6D9uBbWK3oMfbaf93Tnv5riFPtW-2miUTWNr9kGcnd4,228913
-hpcflow_new2-0.2.0a222.dist-info/LICENSE,sha256=Xhxf_KsrJNJFGMogumZhXSTPhUOVHCWf7nU-TDzqg0E,16763
-hpcflow_new2-0.2.0a222.dist-info/METADATA,sha256=93L_wYR_GPmtZ6f2vhlM4QwFdppn3S1p6EjiupYAbx4,2663
-hpcflow_new2-0.2.0a222.dist-info/WHEEL,sha256=kLuE8m1WYU0Ig0_YEGrXyTtiJvKPpLpDEiChiNyei5Y,88
-hpcflow_new2-0.2.0a222.dist-info/entry_points.txt,sha256=aoGtCnFdfPcXfBdu2zZyMOJoz6fPgdR0elqsgrE-USU,106
-hpcflow_new2-0.2.0a222.dist-info/RECORD,,
+hpcflow_new2-0.2.0a227.dist-info/LICENSE,sha256=Xhxf_KsrJNJFGMogumZhXSTPhUOVHCWf7nU-TDzqg0E,16763
+hpcflow_new2-0.2.0a227.dist-info/METADATA,sha256=Q7I7tq_F1uXnwwuKuvhGjdBbBs-AujyfsWgJ4KZ_NFg,2663
+hpcflow_new2-0.2.0a227.dist-info/WHEEL,sha256=kLuE8m1WYU0Ig0_YEGrXyTtiJvKPpLpDEiChiNyei5Y,88
+hpcflow_new2-0.2.0a227.dist-info/entry_points.txt,sha256=aoGtCnFdfPcXfBdu2zZyMOJoz6fPgdR0elqsgrE-USU,106
+hpcflow_new2-0.2.0a227.dist-info/RECORD,,