py-modelrunner 0.19.1__tar.gz → 0.20.0__tar.gz

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/.github/workflows/build.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
-        python-version: ['3.9', '3.13']
+        python-version: ['3.12', '3.14']
 
     steps:
     - uses: actions/checkout@v4

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/.github/workflows/coverage_report.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.9]
+        python-version: [3.12]
 
     steps:
     - uses: actions/checkout@v4

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/.github/workflows/release_pip.yml

@@ -14,7 +14,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.12'
 
     - name: Install dependencies
       run: |

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/.readthedocs.yaml

@@ -9,7 +9,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.9"
+    python: "3.12"
   apt_packages:
     - graphviz
 

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py-modelrunner
-Version: 0.19.1
+Version: 0.20.0
 Summary: Python classes for organizing (HPC) simulations
 Author-email: David Zwicker <david.zwicker@ds.mpg.de>
 License: MIT
@@ -14,23 +14,22 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Utilities
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Python: <3.14,>=3.9
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: <3.15,>=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: jinja2>=3
 Requires-Dist: numpy>=1.18.0
 Requires-Dist: tqdm>=4.45
-Requires-Dist: numcodecs>=0.10
+Requires-Dist: numcodecs>=0.16
 Provides-Extra: all
 Requires-Dist: h5py>=3.5; extra == "all"
 Requires-Dist: pandas>=1.3; extra == "all"
 Requires-Dist: PyYAML>=5; extra == "all"
-Requires-Dist: zarr<3,>=2; extra == "all"
+Requires-Dist: zarr<4,>=3; extra == "all"
 Provides-Extra: test
 Requires-Dist: black>=19; extra == "test"
 Requires-Dist: isort>=5.1; extra == "test"

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/docs/source/installation.rst

@@ -1,7 +1,7 @@
 Installation
 ############
 
-This `py-modelrunner` package is developed for python 3.9+ and should run on all common
+This `py-modelrunner` package is developed for python 3.11+ and should run on all common
 platforms. 
 
 

py_modelrunner-0.20.0/modelrunner/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.20.0'
+__version_tuple__ = version_tuple = (0, 20, 0)
+
+__commit_id__ = commit_id = 'ga172a6158'

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/modelrunner/model/factory.py

@@ -7,7 +7,8 @@ from __future__ import annotations
 
 import functools
 import inspect
-from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, get_args, get_origin
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any, Literal, TypeVar, get_args, get_origin
 
 from ..storage import ModeType
 from .base import ModelBase

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/modelrunner/model/parameters.py

@@ -19,9 +19,9 @@ from __future__ import annotations
 import copy
 import logging
 import warnings
-from collections.abc import Container, Iterator
+from collections.abc import Callable, Container, Iterator
 from dataclasses import dataclass, field
-from typing import Any, Callable, Optional, Union
+from typing import Any
 
 import numpy as np
 
@@ -280,8 +280,8 @@ class HideParameter:
         pass
 
 
-ParameterListType = list[Union[Parameter, HideParameter]]
-ParameterInputType = Optional[dict[str, Any]]
+ParameterListType = list[Parameter | HideParameter]
+ParameterInputType = dict[str, Any] | None
 
 
 class Parameterized:

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/modelrunner/run/compatibility/triage.py

@@ -147,12 +147,12 @@ def _get_format_version(path: Path, label: str) -> int | None:
         store = normalize_zarr_store(path, mode="r")
         if store is None:
             raise RuntimeError
-        with zarr.open_group(store, mode="r") as root:
-            format_version = _find_version(root, label)
-            if format_version is None and label != "data":
-                format_version = _find_version(root, "data")
-                if format_version is not None:
-                    label = "data"
+        root = zarr.open_group(store, mode="r")
+        format_version = _find_version(root, label)
+        if format_version is None and label != "data":
+            format_version = _find_version(root, "data")
+            if format_version is not None:
+                label = "data"
 
     else:
         raise RuntimeError

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/modelrunner/run/compatibility/version1.py

@@ -7,9 +7,8 @@ from __future__ import annotations
 
 import json
 import warnings
-from abc import ABCMeta
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, TypeVar, Union
+from typing import TYPE_CHECKING, Any, TypeVar
 
 import numpy as np
 
@@ -20,7 +19,7 @@ from .triage import guess_format, normalize_zarr_store
 if TYPE_CHECKING:
     import zarr
 
-    zarrElement = Union[zarr.Group, zarr.Array]
+    zarrElement = zarr.Group | zarr.Array
 
 
 class NoData:
@@ -119,7 +118,7 @@ class StateBase:
             raise TypeError(f"Do not know how to load `{cls_name}`")
 
     @classmethod
-    def _from_zarr(cls, zarr_element: zarrElement, *, index=...) -> StateBase:
+    def _from_zarr(cls, zarr_element: zarrElement, *, index=...) -> StateBase:  # type: ignore
         """Create instance of correct subclass from data stored in zarr."""
         # determine the class that knows how to read this data
         cls_name = zarr_element.attrs["__class__"]

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/modelrunner/run/job.py

@@ -366,7 +366,7 @@ def submit_jobs(
 
     # build the list of all varying arguments
     p_vary_list = [
-        dict(zip(p_vary.keys(), values))
+        dict(zip(p_vary.keys(), values, strict=False))
         for values in itertools.product(*p_vary.values())
     ]
 

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/modelrunner/run/launch.py

@@ -9,9 +9,8 @@ import importlib.util
 import inspect
 import logging
 import os.path
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
 from pathlib import Path
-from typing import Callable
 
 from ..model import ModelBase, cleared_default_model, factory, make_model_class
 from .results import Result

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/modelrunner/run/results.py

@@ -411,7 +411,7 @@ class ResultCollection(list[Result]):
         group_values = [self.parameters[name] for name in args]
 
         for group_value in itertools.product(*group_values):
-            group_parameters = dict(zip(args, group_value))
+            group_parameters = dict(zip(args, group_value, strict=False))
             subset = self.filtered(**group_parameters)
             if len(subset) > 0:
                 yield group_parameters, subset

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/modelrunner/storage/access_modes.py

@@ -6,7 +6,7 @@ from __future__ import annotations
 
 import warnings
 from dataclasses import dataclass
-from typing import ClassVar, Literal, Union
+from typing import ClassVar, Literal
 
 FileMode = Literal[
     "r",  # open as readable
@@ -137,7 +137,7 @@ access_truncate = AccessMode(
     dynamic_append=True,
 )
 
-ModeType = Union[str, AccessMode]
+ModeType = str | AccessMode
 
 
 class AccessError(RuntimeError):

py_modelrunner-0.19.1/modelrunner/storage/backend/zarr.py → py_modelrunner-0.20.0/modelrunner/storage/backend/_zarr2.py

@@ -15,13 +15,17 @@ from typing import Any, Union
 import numpy as np
 import zarr
 from numpy.typing import ArrayLike, DTypeLike
-from zarr._storage.store import Store
+
+zarr_version = int(zarr.__version__.split(".", 1)[0])
+assert zarr_version == 2
+
+from zarr.storage import Store
 
 from ..access_modes import ModeType
 from ..attributes import AttrsLike
 from ..base import StorageBase
 
-zarrElement = Union[zarr.Group, zarr.Array]
+zarrElement = zarr.Group | zarr.Array
 
 
 class ZarrStorage(StorageBase):
@@ -56,7 +60,7 @@ class ZarrStorage(StorageBase):
                 if self.mode.file_mode == "r":
                     self._logger.info("DirectoryStore is always opened writable")
 
-                self._store = zarr.DirectoryStore(path)
+                self._store = zarr.storage.LocalStore(path)
 
             elif path.suffix == ".zip":
                 # create a ZipStore
@@ -77,7 +81,7 @@ class ZarrStorage(StorageBase):
                 if self.mode.file_mode == "w" and path.exists():
                     self._logger.info("Delete file `%s`", path)
                     path.unlink()
-                self._store = zarr.SQLiteStore(path)
+                self._store = zarr.storage.SQLiteStore(path)
 
         elif isinstance(store_or_path, Store):
             # use already opened zarr storage
@@ -142,7 +146,7 @@ class ZarrStorage(StorageBase):
             return self._root.keys()  # type: ignore
 
     def is_group(self, loc: Sequence[str], *, ignore_cls: bool = False) -> bool:
-        return isinstance(self[loc], zarr.hierarchy.Group)
+        return isinstance(self[loc], zarr.Group)
 
     def _create_group(self, loc: Sequence[str]) -> None:
         parent, name = self._get_parent(loc)

py_modelrunner-0.20.0/modelrunner/storage/backend/_zarr3.py

@@ -0,0 +1,240 @@
+"""Defines a class storing data in various storages.
+
+Requires the optional :mod:`zarr` module.
+
+.. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import pickle
+import shutil
+from collections.abc import Collection, Sequence
+from pathlib import Path
+from typing import Any, Union
+
+import numpy as np
+import zarr
+from numpy.typing import ArrayLike, DTypeLike
+
+zarr_version = int(zarr.__version__.split(".", 1)[0])
+assert zarr_version == 3
+
+from zarr.abc.store import Store
+
+from ..access_modes import ModeType
+from ..attributes import AttrsLike
+from ..base import StorageBase
+
+zarrElement = zarr.Group | zarr.Array
+
+
+class ZarrStorage(StorageBase):
+    """Storage that stores data in an zarr file or database."""
+
+    extensions = ["zarr", "zip", "sqldb", "lmdb"]
+
+    def __init__(self, store_or_path: str | Path | Store, *, mode: ModeType = "read"):
+        """
+        Args:
+            store_or_path (str or :class:`~pathlib.Path` or :class:`~zarr._storage.store.Store`):
+                File path to the file/folder or a :mod:`zarr` Store
+            mode (str or :class:`~modelrunner.storage.access_modes.AccessMode`):
+                The file mode with which the storage is accessed. Determines allowed
+                operations.
+        """
+        super().__init__(mode=mode)
+
+        if isinstance(store_or_path, (str, Path)):
+            # open zarr storage on file system
+            self._close = True
+            path = Path(store_or_path)
+
+            if self.mode.file_mode == "x" and path.exists():
+                raise FileExistsError(f"File `{path}` already exists")
+
+            if path.suffix in {"", ".zarr"}:
+                # path seems to be a directory or a zarr direction => DirectoryStore
+                if path.is_dir() and self.mode.file_mode == "w":
+                    self._logger.info("Delete directory `{%s}`", path)
+                    shutil.rmtree(path)  # remove the directory to reinstate it
+                if self.mode.file_mode == "r":
+                    self._logger.info("DirectoryStore is always opened writable")
+
+                self._store = zarr.storage.LocalStore(path)
+
+            elif path.suffix == ".zip":
+                # create a ZipStore
+                file_mode = self.mode.file_mode
+                if file_mode == "x":
+                    if path.exists():
+                        raise OSError("File `{path}` already exists")
+                    else:
+                        file_mode = "w"
+                elif file_mode == "w" and path.exists():
+                    self._logger.info("Delete file `%s`", path)
+                    path.unlink()
+
+                self._store = zarr.storage.ZipStore(path, mode=file_mode)
+
+        elif isinstance(store_or_path, Store):
+            # use already opened zarr storage
+            self._close = False
+            self._store = store_or_path
+
+        else:
+            raise TypeError(f"Unknown store `{store_or_path}`")
+
+        zarr_mode = "r" if self.mode.file_mode == "r" else "a"
+        self._root = zarr.open_group(store=self._store, mode=zarr_mode)
+
+    @property
+    def can_update(self) -> bool:
+        """Bool: indicates whether the storage supports updating items."""
+        return not isinstance(self._store, zarr.storage.ZipStore)
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}({self._root.store}, mode="{self.mode.name}")'
+
+    def close(self) -> None:
+        if self._close:
+            self._store.close()
+        self._root = None
+        super().close()
+
+    def _get_parent(self, loc: Sequence[str]) -> tuple[zarr.Group, str]:
+        """Get the parent group for a particular location.
+
+        Args:
+            loc (list of str):
+                The location in the storage where the group will be created
+
+        Returns:
+            (group, str):
+                A tuple consisting of the parent group and the name of the current item
+        """
+        try:
+            path, name = loc[:-1], loc[-1]
+        except IndexError as err:
+            raise KeyError(f"Location `/{'/'.join(loc)}` has no parent") from err
+
+        parent = self._root
+        for part in path:
+            try:
+                parent = parent[part]
+            except KeyError:
+                parent = parent.create_group(part, overwrite=False)
+
+        return parent, name
+
+    def __getitem__(self, loc: Sequence[str]) -> Any:
+        if len(loc) == 0:
+            return self._root
+        else:
+            parent, name = self._get_parent(loc)
+            return parent[name]
+
+    def keys(self, loc: Sequence[str] | None = None) -> Collection[str]:
+        if loc:
+            return self[loc].keys()  # type: ignore
+        else:
+            return self._root.keys()  # type: ignore
+
+    def is_group(self, loc: Sequence[str], *, ignore_cls: bool = False) -> bool:
+        return isinstance(self[loc], zarr.Group)
+
+    def _create_group(self, loc: Sequence[str]) -> None:
+        parent, name = self._get_parent(loc)
+        parent.create_group(name)
+
+    def _read_attrs(self, loc: Sequence[str]) -> AttrsLike:
+        return self[loc].attrs  # type: ignore
+
+    def _write_attr(self, loc: Sequence[str], name: str, value: str) -> None:
+        self[loc].attrs[name] = value
+
+    def _read_array(
+        self, loc: Sequence[str], *, copy: bool, index: int | None = None
+    ) -> np.ndarray:
+        arr_like = self[loc]
+
+        if not isinstance(arr_like, zarr.Array):
+            raise RuntimeError(
+                f"Found {arr_like.__class__} at location `/{'/'.join(loc)}`"
+            )
+
+        is_recarray = arr_like.attrs.get("__recarray__", False)
+        if index is not None:
+            arr_like = arr_like[index]
+
+        # convert it into the right type
+        arr = np.array(arr_like, copy=copy)
+        if is_recarray:
+            arr = arr.view(np.recarray)
+
+        return arr
+
+    def _write_array(self, loc: Sequence[str], arr: np.ndarray) -> None:
+        parent, name = self._get_parent(loc)
+
+        if name in parent:
+            # update an existing array assuming it has the same shape. The credentials
+            # for this operation need to be checked by the caller!
+            parent[name][...] = arr
+
+        else:
+            # create a new array element
+            if arr.dtype == object:
+                el = parent.create_array(name, data=arr, overwrite=True)
+            else:
+                el = parent.create_array(name, data=arr, overwrite=True)
+
+            if isinstance(arr, np.recarray):
+                el.attrs["__recarray__"] = True
+
+    def _create_dynamic_array(
+        self,
+        loc: Sequence[str],
+        shape: tuple[int, ...],
+        dtype: DTypeLike,
+        record_array: bool = False,
+    ) -> None:
+        parent, name = self._get_parent(loc)
+        try:
+            element = parent.zeros(
+                name=name, shape=(0,) + shape, chunks=(1,) + shape, dtype=dtype
+            )
+        except zarr.errors.ContainsArrayError as err:
+            raise RuntimeError(f"Array `/{'/'.join(loc)}` already exists") from err
+        else:
+            if record_array:
+                element.attrs["__recarray__"] = True
+
+    def _extend_dynamic_array(self, loc: Sequence[str], data: ArrayLike) -> None:
+        arr_obj = self[loc]
+        arr_obj.append([data])
+
+    def _read_object(self, loc: Sequence[str]) -> Any:
+        stored = self[loc][0].item()
+        # Check if the object was pickled (starts with pickle marker)
+        if stored.startswith("__pickle__:"):
+            obj_data = stored[11:]  # Remove the marker
+            return pickle.loads(base64.b64decode(obj_data))
+        else:
+            # Otherwise it's JSON
+            return json.loads(stored)
+
+    def _write_object(self, loc: Sequence[str], obj: Any) -> None:
+        # Try JSON serialization first
+        try:
+            obj_enc = json.dumps(obj)
+        except (TypeError, ValueError):
+            # Fall back to pickle for non-JSON-serializable objects
+            pickled = pickle.dumps(obj)
+            obj_enc = "__pickle__:" + base64.b64encode(pickled).decode("ascii")
+
+        data = np.array([obj_enc])
+        parent, name = self._get_parent(loc)
+        parent.create_array(name, data=data, overwrite=True)

py_modelrunner-0.20.0/modelrunner/storage/backend/zarr.py

@@ -0,0 +1,20 @@
+"""Defines a class storing data in various storages.
+
+Requires the optional :mod:`zarr` module.
+
+.. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
+"""
+
+from __future__ import annotations
+
+import zarr
+
+zarr_version = int(zarr.__version__.split(".", 1)[0])
+
+if zarr_version == 2:
+    # import classes from paths of zarr version 2
+    from ._zarr2 import ZarrStorage
+
+elif zarr_version == 3:
+    # import classes from paths of zarr version 3
+    from ._zarr3 import ZarrStorage  # type: ignore

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/modelrunner/storage/tools.py

@@ -6,7 +6,6 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import Union
 
 from .access_modes import AccessMode
 from .backend import AVAILABLE_STORAGE, MemoryStorage
@@ -14,7 +13,7 @@ from .base import StorageBase
 from .group import StorageGroup
 from .utils import Location
 
-StorageID = Union[None, str, Path, StorageGroup, StorageBase]
+StorageID = str | Path | StorageGroup | StorageBase | None
 
 
 class open_storage(StorageGroup):
@@ -121,7 +120,7 @@ class open_storage(StorageGroup):
 
             if isinstance(storage, zarr.Group):
                 self._close = False
-                store_obj = ZarrStorage(storage._store, **kwargs)
+                store_obj = ZarrStorage(storage.store, **kwargs)
                 loc = [storage.path] + [loc]
 
         if store_obj is None:

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/modelrunner/storage/trajectory.py

@@ -124,9 +124,7 @@ class TrajectoryWriter:
                 shape: tuple[int, ...] = data.shape
                 self._item_type = "array"
             else:
-                dtype = object
-                shape = ()
-                self._item_type = "object"
+                raise NotImplementedError("Trajectories must be arrays for zarr3")
             self._trajectory.create_dynamic_array("data", shape=shape, dtype=dtype)
             self._trajectory.create_dynamic_array("time", shape=(), dtype=float)
             self._trajectory.write_attrs(None, {"item_type": self._item_type})
@@ -138,10 +136,10 @@ class TrajectoryWriter:
 
         if self._item_type == "array":
             self._trajectory.extend_dynamic_array("data", data)
-        elif self._item_type == "object":
-            arr: np.ndarray = np.empty((), dtype=object)
-            arr[...] = data
-            self._trajectory.extend_dynamic_array("data", arr)
+        # elif self._item_type == "object":
+        #     arr: np.ndarray = np.empty((), dtype=object)
+        #     arr[...] = data
+        #     self._trajectory.extend_dynamic_array("data", arr)
         else:
             raise NotImplementedError
         self._trajectory.extend_dynamic_array("time", time)

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/modelrunner/storage/utils.py

@@ -9,9 +9,9 @@ import codecs
 import inspect
 import pickle
 from collections import defaultdict
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
 from importlib import import_module
-from typing import TYPE_CHECKING, Any, Callable, Literal, Union, overload
+from typing import TYPE_CHECKING, Any, Literal, overload
 
 import numpy as np
 
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
 PICKLE_PROTOCOL = pickle.HIGHEST_PROTOCOL
 
 
-Location = Union[None, str, Sequence["Location"]]
+Location = str | Sequence["Location"] | None
 
 
 @overload

{py_modelrunner-0.19.1 → py_modelrunner-0.20.0}/modelrunner/utils.py

@@ -73,8 +73,9 @@ def import_class(identifier: str):
 def is_serial_or_mpi_root() -> bool:
     """Function checking whether the current program is serial or an MPI root node."""
     try:
-        from mpi4py import MPI
+        from mpi4py import MPI  # type: ignore
     except ImportError:
         return True  # assume we are in a serial run
     else:
-        return MPI.COMM_WORLD.rank == 0  # check whether we are the root node
+        # check whether we are the root node
+        return MPI.COMM_WORLD.rank == 0  # type: ignore