py-modelrunner 0.15.0__tar.gz → 0.16.0__tar.gz

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py-modelrunner
-Version: 0.15.0
+Version: 0.16.0
 Summary: Python classes for organizing (HPC) simulations
 Author-email: David Zwicker <david.zwicker@ds.mpg.de>
 License: MIT

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/docs/requirements.txt

@@ -1,4 +1,4 @@
--r ../requirements.txt
+-r ../requirements_full.txt
 Sphinx>=4
 sphinx-autodoc-annotation>=1.0
 sphinx-gallery>=0.6

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/docs/source/.gitignore

@@ -1 +1,2 @@
 /packages/
+/examples/

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/docs/source/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # Configuration file for the Sphinx documentation builder.
 #
@@ -226,8 +225,8 @@ napoleon_numpy_docstring = False
 napoleon_include_init_with_doc = False
 napoleon_include_private_with_doc = False
 napoleon_include_special_with_doc = False
-napoleon_use_admonition_for_examples = False
-napoleon_use_admonition_for_notes = False
+napoleon_use_admonition_for_examples = True
+napoleon_use_admonition_for_notes = True
 napoleon_use_admonition_for_references = False
 napoleon_use_ivar = False
 napoleon_use_param = True
@@ -239,13 +238,16 @@ napoleon_custom_sections = None
 
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {
-    "https://docs.python.org/3/": None,
-    "https://docs.scipy.org/doc/numpy/": None,
-    "https://docs.scipy.org/doc/scipy/reference": None,
-    "https://matplotlib.org": None,
-    "http://docs.h5py.org/en/latest/": None,
-    "https://py-pde.readthedocs.io/en/latest": None,
-    "https://py-droplets.readthedocs.io/en/latest": None,
+    "h5py": ("https://docs.h5py.org/en/latest", None),
+    "matplotlib": ("https://matplotlib.org/stable", None),
+    "modelrunner": ("https://py-modelrunner.readthedocs.io/en/latest", None),
+    "napari": ("https://napari.org/", None),
+    "numba": ("https://numba.pydata.org/numba-doc/latest/", None),
+    "numpy": ("https://numpy.org/doc/stable", None),
+    "pde": ("https://py-pde.readthedocs.io/en/latest", None),
+    "python": ("https://docs.python.org/3/", None),
+    "scipy": ("https://docs.scipy.org/doc/scipy/", None),
+    "sympy": ("https://docs.sympy.org/latest/", None),
 }
 
 
@@ -255,6 +257,11 @@ graphviz_output_format = "svg"
 # for the graph, which we here corrected manually, but which lead to issues if we also
 # used the inheritance_diagram extension
 
+# create all example files
+from parse_examples import main
+
+main()
+
 # run autodoc
 autodoc_typehints = "description"
 

py-modelrunner-0.16.0/docs/source/examples.rst

@@ -0,0 +1,10 @@
+Examples
+########
+
+The following examples showcase some functionality of the package
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    examples/* 

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/docs/source/index.rst

@@ -12,11 +12,13 @@ Contents
 
 .. toctree::
     :maxdepth: 2
+    :glob:
 
     installation
-    structure
+    user_manual
+    examples
     packages/modelrunner
- 
+
 
 Indices and tables
 ------------------

py-modelrunner-0.16.0/docs/source/parse_examples.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+
+import pathlib
+import re
+
+# Root direcotry of the package
+ROOT = pathlib.Path(__file__).absolute().parents[2]
+# directory where all the examples reside
+INPUT = ROOT / "examples"
+# directory to which the documents are writen
+OUTPUT = ROOT / "docs" / "source" / "examples"
+
+
+def main():
+    """parse all examples and write them in a special example module"""
+    # create the output directory
+    OUTPUT.mkdir(parents=True, exist_ok=True)
+
+    regex = r'"""((?!""").*?)"""'
+
+    # iterate over all examples
+    for path_in in INPUT.glob("*.py"):
+        path_out = OUTPUT / (path_in.stem + ".rst")
+        print(f"Found example {path_in}")
+        with path_in.open("r") as file_in, path_out.open("w") as file_out:
+            # write the header for the rst file
+            file_out.write(path_in.name + "\n")
+            file_out.write("-" * len(path_in.name) + "\n\n")
+
+            # read all content
+            file_content = file_in.read()
+
+            # identify shebang
+            if file_content.startswith("#"):
+                shebang, file_content = file_content.split("\n", 1)
+                if shebang == "#!/usr/bin/env python3":
+                    shebang = ""  # filter trivial case
+            else:
+                shebang = None
+
+            # locate file docstring
+            matches = re.search(regex, file_content, re.DOTALL)
+            if matches:
+                for line in matches.group(1).split("\n"):
+                    if ".. codeauthor::" in line:
+                        continue
+                    file_out.write(f"{line}\n")
+
+                file_content = file_content[matches.end(0) :]
+
+            # write actual code
+            file_out.write(".. code-block:: python\n\n")
+            if shebang:
+                file_out.write(f"    {shebang}\n\n")
+            header = True
+            for line in file_content.split("\n"):
+                # skip the shebang, comments and empty lines in the beginning
+                if header and (line.startswith("#") or len(line.strip()) == 0):
+                    continue
+                header = False  # first real line was reached
+                file_out.write(f"    {line}\n")
+
+
+if __name__ == "__main__":
+    main()

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/docs/source/run_autodoc.py

@@ -27,7 +27,7 @@ def replace_in_file(infile, replacements, outfile=None):
     if outfile is None:
         outfile = infile
 
-    with open(infile, "r") as fp:
+    with open(infile) as fp:
         content = fp.read()
 
     for key, value in replacements.items():

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/docs/source/sphinx_simplify_typehints.py

@@ -83,7 +83,7 @@ REPLACEMENTS = collections.OrderedDict(
 # replacement rules based on regular expressions
 REPLACEMENTS_REGEX = {
     # remove full package path and only leave the module/class identifier
-    "multicomp\.(\w+\.)*": "",
+    r"multicomp\.(\w+\.)*": "",
 }
 
 

py-modelrunner-0.16.0/docs/source/user_manual.rst

@@ -0,0 +1,53 @@
+User Manual
+===========
+
+Main structure of the package
+-----------------------------
+
+The package offers several different components that can be used separately or together:
+
+Hierarchical input/output using :mod:`~modelrunner.storage`:
+    The module provides an abstract interface for writing and reading data using an
+    hierarchical organization.
+    Storages are conveniently created by :func:`~modelrunner.storage.tools.open_storage`,
+    which returns a storage object that offers a dict-like interface.
+
+Defining parameters using :mod:`~modelrunner.parameters`:
+    The module provides the :class:`~modelrunner.parameters.Parameter` class, describing a single parameter. 
+    This can be used together with the mixin :class:`~modelrunner.parameters.Parametrized`,
+    which allows to equip classes with default parameters and some convenience methods.
+
+Defining models using :mod:`~modelrunner.model`:
+    Models are augmented functions, which define input, calculations, and output.
+    Models can be conveniently created by decorating a function or by subclassing
+    :class:`~modelrunner.model.ModelBase`, which is built on the parameter classes.
+
+Model results are captured by :mod:`~modelrunner.results`:
+    Results are returned as the special :class:`~modelrunner.results.Result`, which
+    keeps track of the input parameters and the data calculated by the model.
+    :class:`~modelrunner.results.ResultCollection` describes a collections of the same
+    model evoked with different parameter values.
+
+Submitting models to HPC using :mod:`~modelrunner.run`:
+    A single model can be submitted to a compute node using :func:`~modelrunner.run.job.submit_job`,
+    e.g., to run the computation on a high performance compute cluster.
+    A parameter study using multiple jobs can be conveniently submitted using :func:`~modelrunner.run.job.submit_jobs`.
+    The results written to one directory can then be conveniently analyzed using :meth:`~modelrunner.results.ResultCollection.from_folder`.
+
+
+Design philosophy
+-----------------
+
+The main requirements for the storage classes were
+
+- *Usability*: The user should not need to think about how data is stored in different files
+- *Flexibility*: We want a general interface to write data in multiple file formats (YAML, HDF, zarr, ...)
+- *Stability*: Future versions of the package should be able to read older files even when the internal definitions of file formats change
+- *Extensibility*: States should be subclasses to implement more complicated requirements (e.g., particular serialization)
+- *Self-explainability*: The files should in principle contain all information to reconstruct the data, even if the `py-modelrunner` package is no longer available.
+- *Efficiency*: The files should only store necessary information.
+
+The last point results in particular constraints if we want to store temporal simulation results.
+In most cases, there are are some data that are kept fixed for the simulation (describing physical parameters) and others that evolve with time.
+We denote by `attributes` the parameters that are kept fixed and by `data` the data that varies over time.
+The state classes are already prepared to deal with such data, in conjuction with the :mod:`~modelrunner.storage.trajectory` module.

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/modelrunner/__init__.py

@@ -2,7 +2,6 @@
 .. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
 """
 
-
 # determine the package version
 try:
     # try reading version of the automatically generated module

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/modelrunner/__main__.py

@@ -4,6 +4,8 @@ Main module allowing to use the package to wrap existing code
 .. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
 """
 
+from __future__ import annotations
+
 import sys
 
 from modelrunner import Result, run_script

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/modelrunner/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.15.0'
-__version_tuple__ = version_tuple = (0, 15, 0)
+__version__ = version = '0.16.0'
+__version_tuple__ = version_tuple = (0, 16, 0)

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/modelrunner/compatibility/triage.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 import json
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Mapping, Optional, Union
+from typing import TYPE_CHECKING, Any, Mapping, Union
 
 from ..model import ModelBase
 from ..results import Result
@@ -42,7 +42,7 @@ def guess_format(path: Path) -> str:
         return "zarr"  # fallback to the default storage method based on zarr
 
 
-def normalize_zarr_store(store: Store, mode: str = "a") -> Optional[Store]:
+def normalize_zarr_store(store: Store, mode: str = "a") -> Store | None:
     """determine best file format for zarr storage
 
     In particular, we use a :class:`~zarr.storage.ZipStore` when a path looking like a
@@ -70,7 +70,7 @@ def normalize_zarr_store(store: Store, mode: str = "a") -> Optional[Store]:
     return store
 
 
-def _find_version(data: Mapping[str, Any], label: str) -> Optional[int]:
+def _find_version(data: Mapping[str, Any], label: str) -> int | None:
     """try finding version information in different places in `data`
 
     Args:
@@ -83,7 +83,7 @@ def _find_version(data: Mapping[str, Any], label: str) -> Optional[int]:
         int: The format version or None if it could not be found
     """
 
-    def read_version(item) -> Optional[str]:
+    def read_version(item) -> str | None:
         """try reading attribute from a particular item"""
         if hasattr(item, "attrs"):
             return read_version(item.attrs)
@@ -111,8 +111,8 @@ def _find_version(data: Mapping[str, Any], label: str) -> Optional[int]:
 
 
 def result_check_load_old_version(
-    path: Path, loc: str, *, model: Optional[ModelBase] = None
-) -> Optional[Result]:
+    path: Path, loc: str, *, model: ModelBase | None = None
+) -> Result | None:
     """check whether the resource can be loaded with an older version of the package
 
     Args:
@@ -132,13 +132,13 @@ def result_check_load_old_version(
     # check for compatibility
     fmt = guess_format(path)
     if fmt == "json":
-        with open(path, mode="r") as fp:
+        with open(path) as fp:
             format_version = _find_version(json.load(fp), label)
 
     elif fmt == "yaml":
         import yaml
 
-        with open(path, mode="r") as fp:
+        with open(path) as fp:
             format_version = _find_version(yaml.safe_load(fp), label)
 
     elif fmt == "hdf":

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/modelrunner/compatibility/version0.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 import json
 from pathlib import Path
-from typing import Any, Mapping, Optional
+from typing import Any, Mapping
 
 import numpy as np
 
@@ -32,7 +32,7 @@ def read_hdf_data(node):
 
 
 def _Result_from_simple_objects(
-    content: Mapping[str, Any], model: Optional[ModelBase] = None
+    content: Mapping[str, Any], model: ModelBase | None = None
 ) -> Result:
     """read result from simple object (like loaded from a JSON file) using version 0
 
@@ -54,7 +54,7 @@ def _Result_from_simple_objects(
     )
 
 
-def _Result_from_hdf(hdf_element, model: Optional[ModelBase] = None) -> Result:
+def _Result_from_hdf(hdf_element, model: ModelBase | None = None) -> Result:
     """old reader for backward compatible reading"""
     model_data = {key: json.loads(value) for key, value in hdf_element.attrs.items()}
     if "result" in hdf_element:
@@ -80,14 +80,14 @@ def result_from_file_v0(path: Path, **kwargs) -> Result:
     """
     fmt = guess_format(path)
     if fmt == "json":
-        with open(path, mode="r") as fp:
+        with open(path) as fp:
             content = json.load(fp)
         return _Result_from_simple_objects(content, **kwargs)
 
     elif fmt == "yaml":
         import yaml
 
-        with open(path, mode="r") as fp:
+        with open(path) as fp:
             content = yaml.safe_load(fp)
         return _Result_from_simple_objects(content, **kwargs)
 

{py-modelrunner-0.15.0 → py-modelrunner-0.16.0}/modelrunner/compatibility/version1.py

@@ -10,7 +10,7 @@ import json
 import warnings
 from abc import ABCMeta
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Optional, Type, TypeVar, Union
+from typing import TYPE_CHECKING, Any, TypeVar, Union
 
 import numpy as np
 
@@ -55,10 +55,10 @@ class StateBase(metaclass=ABCMeta):
     _state_format_version = 2
     """int: number indicating the version of the file format"""
 
-    _state_classes: Dict[str, Type[StateBase]] = {}
+    _state_classes: dict[str, type[StateBase]] = {}
     """dict: class-level list of all subclasses of StateBase"""
 
-    def _state_init(self, attributes: Dict[str, Any], data=NoData) -> None:
+    def _state_init(self, attributes: dict[str, Any], data=NoData) -> None:
         """initialize the state with attributes and (optionally) data
 
         Args:
@@ -71,7 +71,7 @@ class StateBase(metaclass=ABCMeta):
             self._state_attributes = attributes
 
     @classmethod
-    def from_data(cls: Type[TState], attributes: Dict[str, Any], data=NoData) -> TState:
+    def from_data(cls: type[TState], attributes: dict[str, Any], data=NoData) -> TState:
         """create instance of any state class from attributes and data
 
         Args:
@@ -84,7 +84,8 @@ class StateBase(metaclass=ABCMeta):
         # copy attributes since they are modified in this function
         attributes = attributes.copy()
         cls_name = attributes["__class__"]
-        assert cls_name == cls.__name__
+        if cls_name != cls.__name__:
+            raise RuntimeError(f"Expected `{cls.__name__}` but data had `{cls_name}`")
 
         format_version = attributes.pop("__version__", 0)
         if format_version != 1:
@@ -97,7 +98,7 @@ class StateBase(metaclass=ABCMeta):
         return obj
 
     @classmethod
-    def _from_simple_objects(cls, content: Dict[str, Any]) -> StateBase:
+    def _from_simple_objects(cls, content: dict[str, Any]) -> StateBase:
         """create state from text data
 
         Args:
@@ -115,7 +116,7 @@ class StateBase(metaclass=ABCMeta):
             return ArrayCollectionState.from_data(content["attributes"], col_data)
 
         elif cls_name == "DictState":
-            dict_data: Dict[str, StateBase] = {}
+            dict_data: dict[str, StateBase] = {}
             for label, substate in content["data"].items():
                 dict_data[label] = cls._from_simple_objects(substate)
 
@@ -128,7 +129,7 @@ class StateBase(metaclass=ABCMeta):
             raise TypeError(f"Do not know how to load `{cls_name}`")
 
     @classmethod
-    def _from_zarr(cls, zarr_element: "zarrElement", *, index=...) -> StateBase:
+    def _from_zarr(cls, zarr_element: zarrElement, *, index=...) -> StateBase:
         """create instance of correct subclass from data stored in zarr"""
         # determine the class that knows how to read this data
         cls_name = zarr_element.attrs["__class__"]
@@ -163,24 +164,20 @@ class StateBase(metaclass=ABCMeta):
             raise TypeError(f"Do not know how to load `{cls_name}`")
 
 
-class ArrayState(StateBase):
-    ...
+class ArrayState(StateBase): ...
 
 
-class ArrayCollectionState(StateBase):
-    ...
+class ArrayCollectionState(StateBase): ...
 
 
-class DictState(StateBase):
-    ...
+class DictState(StateBase): ...
 
 
-class ObjectState(StateBase):
-    ...
+class ObjectState(StateBase): ...
 
 
 def _Result_from_simple_objects(
-    content: Dict[str, Any], model: Optional[ModelBase] = None
+    content: dict[str, Any], model: ModelBase | None = None
 ) -> Result:
     """read result from simple object (like loaded from a JSON file) using version 1
 
@@ -202,7 +199,7 @@ def _Result_from_simple_objects(
 
 
 def _Result_from_zarr(
-    zarr_element: "zarrElement", *, index=..., model: Optional[ModelBase] = None
+    zarr_element: zarrElement, *, index=..., model: ModelBase | None = None
 ) -> Result:
     """create result from data stored in zarr"""
     attributes = {key: json.loads(value) for key, value in zarr_element.attrs.items()}
@@ -236,14 +233,14 @@ def result_from_file_v1(store: Path, *, label: str = "data", **kwargs) -> Result
     """
     fmt = guess_format(store)
     if fmt == "json":
-        with open(store, mode="r") as fp:
+        with open(store) as fp:
             content = json.load(fp)
         return _Result_from_simple_objects(content, **kwargs)
 
     elif fmt == "yaml":
         import yaml
 
-        with open(store, mode="r") as fp:
+        with open(store) as fp:
             content = yaml.safe_load(fp)
         return _Result_from_simple_objects(content, **kwargs)
 

py-modelrunner-0.16.0/modelrunner/config.py

@@ -0,0 +1,149 @@
+"""
+Handles configuration variables
+
+.. autosummary::
+   :nosignatures:
+
+   Config
+
+.. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
+"""
+
+from __future__ import annotations
+
+import collections
+import contextlib
+from pathlib import Path
+from typing import Any, Sequence
+
+from .parameters import DeprecatedParameter, Parameter
+
+
+class Config(collections.UserDict):
+    """class handling the package configuration"""
+
+    def __init__(
+        self,
+        default: Sequence[Parameter] | None = None,
+        mode: str = "update",
+        *,
+        check_validity: bool = True,
+        include_deprecated: bool = True,
+    ):
+        """
+        Args:
+            default (sequence of :class:`~modelrunner.parameters.Parameter`, optional):
+                Default configuration values. The default configuration also defines
+                what parameters can typically be defined and it provides additional
+                information for these parameters.
+            mode (str):
+                Defines the mode in which the configuration is used. Possible values are
+
+                * `insert`: any new configuration key can be inserted
+                * `update`: only the values defined by `default` can be updated
+                * `locked`: no values can be changed
+
+                Note that the items specified by `items` will always be inserted,
+                independent of the `mode`.
+            check_validity (bool):
+                Determines whether a `ValueError` is raised if there are keys in
+                parameters that are not in the defaults. If `False`, additional items
+                are simply stored in `self.parameters`
+            include_deprecated (bool):
+                Include deprecated parameters
+        """
+        if default is None:
+            default = []
+        self._default = default
+
+        # initialize parameters with default ones
+        self.mode = "insert"  # temporarily allow inserting items
+        super().__init__()
+        for param_obj in default:
+            if include_deprecated or not isinstance(param_obj, DeprecatedParameter):
+                self[param_obj.name] = param_obj.convert(strict=check_validity)
+
+        # set the mode for future additions
+        self.mode = mode
+
+    def load(self, path: str | Path):
+        """load configuration from yaml file"""
+        import yaml
+
+        with open(path) as fp:
+            self.update(yaml.safe_load(fp))
+
+    def save(self, path: str | Path):
+        """save configuration to yaml file"""
+        import yaml
+
+        with open(path, "w") as fp:
+            yaml.dump(self.to_dict(), fp)
+
+    def __getitem__(self, key: str):
+        """retrieve item `key`"""
+        parameter = self.data[key]
+        if isinstance(parameter, Parameter):
+            return parameter.convert()
+        else:
+            return parameter
+
+    def __setitem__(self, key: str, value):
+        """update item `key` with `value`"""
+        if self.mode == "insert":
+            self.data[key] = value
+
+        elif self.mode == "update":
+            if key not in self:
+                raise KeyError(
+                    f"{key} is not present, but config is in `{self.mode}` mode"
+                )
+            self.data[key] = value
+
+        elif self.mode == "locked":
+            raise RuntimeError("Configuration is locked")
+
+        else:
+            raise ValueError(f"Unsupported configuration mode `{self.mode}`")
+
+    def __delitem__(self, key: str):
+        """removes item `key`"""
+        if self.mode == "insert":
+            del self.data[key]
+        else:
+            raise RuntimeError("Configuration is not in `insert` mode")
+
+    def copy(self) -> Config:
+        """return a copy of the configuration"""
+        obj = self.__class__(self._default, self.mode)
+        obj.update(self)
+        return obj
+
+    def to_dict(self) -> dict[str, Any]:
+        """convert the configuration to a simple dictionary
+
+        Returns:
+            dict: A representation of the configuration in a normal :class:`dict`.
+        """
+        return {k: v for k, v in self.items()}
+
+    def __repr__(self) -> str:
+        """represent the configuration as a string"""
+        return f"{self.__class__.__name__}({repr(self.to_dict())})"
+
+    @contextlib.contextmanager
+    def __call__(self, values: dict[str, Any] | None = None, **kwargs):
+        """context manager temporarily changing the configuration
+
+        Args:
+            values (dict): New configuration parameters
+            **kwargs: New configuration parameters
+        """
+        data_initial = self.to_dict()  # save old configuration
+        # set new configuration
+        if values is not None:
+            self.update(values)
+        self.update(kwargs)
+        yield  # return to caller
+        # restore old configuration
+        self.update(data_initial)