py-modelrunner 0.14.0__tar.gz → 0.16.0__tar.gz

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/.github/workflows/build.yml

@@ -9,20 +9,27 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
-        python-version: ['3.8', '3.11']
+        python-version: ['3.8', '3.12']
 
     steps:
     - uses: actions/checkout@v3
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
         pip install -r tests/requirements.txt
 
+    - name: Install optional dependencies
+      continue-on-error: true
+      run: |
+        pip install py-pde
+
     - name: Test with pytest
       env:
         NUMBA_WARNINGS: 1

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py-modelrunner
-Version: 0.14.0
+Version: 0.16.0
 Summary: Python classes for organizing (HPC) simulations
 Author-email: David Zwicker <david.zwicker@ds.mpg.de>
 License: MIT
@@ -18,7 +18,8 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: <3.12,>=3.8
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: <3.13,>=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: jinja2>=3

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/docs/requirements.txt

@@ -1,4 +1,4 @@
--r ../requirements.txt
+-r ../requirements_full.txt
 Sphinx>=4
 sphinx-autodoc-annotation>=1.0
 sphinx-gallery>=0.6

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/docs/source/.gitignore

@@ -1 +1,2 @@
 /packages/
+/examples/

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/docs/source/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # Configuration file for the Sphinx documentation builder.
 #
@@ -226,8 +225,8 @@ napoleon_numpy_docstring = False
 napoleon_include_init_with_doc = False
 napoleon_include_private_with_doc = False
 napoleon_include_special_with_doc = False
-napoleon_use_admonition_for_examples = False
-napoleon_use_admonition_for_notes = False
+napoleon_use_admonition_for_examples = True
+napoleon_use_admonition_for_notes = True
 napoleon_use_admonition_for_references = False
 napoleon_use_ivar = False
 napoleon_use_param = True
@@ -239,13 +238,16 @@ napoleon_custom_sections = None
 
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {
-    "https://docs.python.org/3/": None,
-    "https://docs.scipy.org/doc/numpy/": None,
-    "https://docs.scipy.org/doc/scipy/reference": None,
-    "https://matplotlib.org": None,
-    "http://docs.h5py.org/en/latest/": None,
-    "https://py-pde.readthedocs.io/en/latest": None,
-    "https://py-droplets.readthedocs.io/en/latest": None,
+    "h5py": ("https://docs.h5py.org/en/latest", None),
+    "matplotlib": ("https://matplotlib.org/stable", None),
+    "modelrunner": ("https://py-modelrunner.readthedocs.io/en/latest", None),
+    "napari": ("https://napari.org/", None),
+    "numba": ("https://numba.pydata.org/numba-doc/latest/", None),
+    "numpy": ("https://numpy.org/doc/stable", None),
+    "pde": ("https://py-pde.readthedocs.io/en/latest", None),
+    "python": ("https://docs.python.org/3/", None),
+    "scipy": ("https://docs.scipy.org/doc/scipy/", None),
+    "sympy": ("https://docs.sympy.org/latest/", None),
 }
 
 
@@ -255,6 +257,11 @@ graphviz_output_format = "svg"
 # for the graph, which we here corrected manually, but which lead to issues if we also
 # used the inheritance_diagram extension
 
+# create all example files
+from parse_examples import main
+
+main()
+
 # run autodoc
 autodoc_typehints = "description"
 

py-modelrunner-0.16.0/docs/source/examples.rst

@@ -0,0 +1,10 @@
+Examples
+########
+
+The following examples showcase some functionality of the package
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    examples/* 

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/docs/source/index.rst

@@ -12,11 +12,13 @@ Contents
 
 .. toctree::
     :maxdepth: 2
+    :glob:
 
     installation
-    structure
+    user_manual
+    examples
     packages/modelrunner
- 
+
 
 Indices and tables
 ------------------

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/docs/source/installation.rst

@@ -26,7 +26,7 @@ You can thus install it using
 
     conda install -c conda-forge py-modelrunner
 
-This installation includes all required dependencies to have all features of `py-pde`.
+This installation includes all required dependencies to have all features of `py-modelrunner`.
 
 
 Installing from source

py-modelrunner-0.16.0/docs/source/parse_examples.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+
+import pathlib
+import re
+
+# Root direcotry of the package
+ROOT = pathlib.Path(__file__).absolute().parents[2]
+# directory where all the examples reside
+INPUT = ROOT / "examples"
+# directory to which the documents are writen
+OUTPUT = ROOT / "docs" / "source" / "examples"
+
+
+def main():
+    """parse all examples and write them in a special example module"""
+    # create the output directory
+    OUTPUT.mkdir(parents=True, exist_ok=True)
+
+    regex = r'"""((?!""").*?)"""'
+
+    # iterate over all examples
+    for path_in in INPUT.glob("*.py"):
+        path_out = OUTPUT / (path_in.stem + ".rst")
+        print(f"Found example {path_in}")
+        with path_in.open("r") as file_in, path_out.open("w") as file_out:
+            # write the header for the rst file
+            file_out.write(path_in.name + "\n")
+            file_out.write("-" * len(path_in.name) + "\n\n")
+
+            # read all content
+            file_content = file_in.read()
+
+            # identify shebang
+            if file_content.startswith("#"):
+                shebang, file_content = file_content.split("\n", 1)
+                if shebang == "#!/usr/bin/env python3":
+                    shebang = ""  # filter trivial case
+            else:
+                shebang = None
+
+            # locate file docstring
+            matches = re.search(regex, file_content, re.DOTALL)
+            if matches:
+                for line in matches.group(1).split("\n"):
+                    if ".. codeauthor::" in line:
+                        continue
+                    file_out.write(f"{line}\n")
+
+                file_content = file_content[matches.end(0) :]
+
+            # write actual code
+            file_out.write(".. code-block:: python\n\n")
+            if shebang:
+                file_out.write(f"    {shebang}\n\n")
+            header = True
+            for line in file_content.split("\n"):
+                # skip the shebang, comments and empty lines in the beginning
+                if header and (line.startswith("#") or len(line.strip()) == 0):
+                    continue
+                header = False  # first real line was reached
+                file_out.write(f"    {line}\n")
+
+
+if __name__ == "__main__":
+    main()

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/docs/source/run_autodoc.py

@@ -27,7 +27,7 @@ def replace_in_file(infile, replacements, outfile=None):
     if outfile is None:
         outfile = infile
 
-    with open(infile, "r") as fp:
+    with open(infile) as fp:
         content = fp.read()
 
     for key, value in replacements.items():

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/docs/source/sphinx_simplify_typehints.py

@@ -83,7 +83,7 @@ REPLACEMENTS = collections.OrderedDict(
 # replacement rules based on regular expressions
 REPLACEMENTS_REGEX = {
     # remove full package path and only leave the module/class identifier
-    "multicomp\.(\w+\.)*": "",
+    r"multicomp\.(\w+\.)*": "",
 }
 
 

py-modelrunner-0.16.0/docs/source/user_manual.rst

@@ -0,0 +1,53 @@
+User Manual
+===========
+
+Main structure of the package
+-----------------------------
+
+The package offers several different components that can be used separately or together:
+
+Hierarchical input/output using :mod:`~modelrunner.storage`:
+    The module provides an abstract interface for writing and reading data using an
+    hierarchical organization.
+    Storages are conveniently created by :func:`~modelrunner.storage.tools.open_storage`,
+    which returns a storage object that offers a dict-like interface.
+
+Defining parameters using :mod:`~modelrunner.parameters`:
+    The module provides the :class:`~modelrunner.parameters.Parameter` class, describing a single parameter. 
+    This can be used together with the mixin :class:`~modelrunner.parameters.Parametrized`,
+    which allows to equip classes with default parameters and some convenience methods.
+
+Defining models using :mod:`~modelrunner.model`:
+    Models are augmented functions, which define input, calculations, and output.
+    Models can be conveniently created by decorating a function or by subclassing
+    :class:`~modelrunner.model.ModelBase`, which is built on the parameter classes.
+
+Model results are captured by :mod:`~modelrunner.results`:
+    Results are returned as the special :class:`~modelrunner.results.Result`, which
+    keeps track of the input parameters and the data calculated by the model.
+    :class:`~modelrunner.results.ResultCollection` describes a collections of the same
+    model evoked with different parameter values.
+
+Submitting models to HPC using :mod:`~modelrunner.run`:
+    A single model can be submitted to a compute node using :func:`~modelrunner.run.job.submit_job`,
+    e.g., to run the computation on a high performance compute cluster.
+    A parameter study using multiple jobs can be conveniently submitted using :func:`~modelrunner.run.job.submit_jobs`.
+    The results written to one directory can then be conveniently analyzed using :meth:`~modelrunner.results.ResultCollection.from_folder`.
+
+
+Design philosophy
+-----------------
+
+The main requirements for the storage classes were
+
+- *Usability*: The user should not need to think about how data is stored in different files
+- *Flexibility*: We want a general interface to write data in multiple file formats (YAML, HDF, zarr, ...)
+- *Stability*: Future versions of the package should be able to read older files even when the internal definitions of file formats change
+- *Extensibility*: States should be subclasses to implement more complicated requirements (e.g., particular serialization)
+- *Self-explainability*: The files should in principle contain all information to reconstruct the data, even if the `py-modelrunner` package is no longer available.
+- *Efficiency*: The files should only store necessary information.
+
+The last point results in particular constraints if we want to store temporal simulation results.
+In most cases, there are are some data that are kept fixed for the simulation (describing physical parameters) and others that evolve with time.
+We denote by `attributes` the parameters that are kept fixed and by `data` the data that varies over time.
+The state classes are already prepared to deal with such data, in conjuction with the :mod:`~modelrunner.storage.trajectory` module.

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/modelrunner/__init__.py

@@ -2,7 +2,6 @@
 .. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
 """
 
-
 # determine the package version
 try:
     # try reading version of the automatically generated module

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/modelrunner/__main__.py

@@ -4,6 +4,8 @@ Main module allowing to use the package to wrap existing code
 .. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
 """
 
+from __future__ import annotations
+
 import sys
 
 from modelrunner import Result, run_script

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/modelrunner/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.14.0'
-__version_tuple__ = version_tuple = (0, 14, 0)
+__version__ = version = '0.16.0'
+__version_tuple__ = version_tuple = (0, 16, 0)

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/modelrunner/compatibility/triage.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 import json
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Mapping, Optional, Union
+from typing import TYPE_CHECKING, Any, Mapping, Union
 
 from ..model import ModelBase
 from ..results import Result
@@ -42,7 +42,7 @@ def guess_format(path: Path) -> str:
         return "zarr"  # fallback to the default storage method based on zarr
 
 
-def normalize_zarr_store(store: Store, mode: str = "a") -> Optional[Store]:
+def normalize_zarr_store(store: Store, mode: str = "a") -> Store | None:
     """determine best file format for zarr storage
 
     In particular, we use a :class:`~zarr.storage.ZipStore` when a path looking like a
@@ -70,7 +70,7 @@ def normalize_zarr_store(store: Store, mode: str = "a") -> Optional[Store]:
     return store
 
 
-def _find_version(data: Mapping[str, Any], label: str) -> Optional[int]:
+def _find_version(data: Mapping[str, Any], label: str) -> int | None:
     """try finding version information in different places in `data`
 
     Args:
@@ -83,7 +83,7 @@ def _find_version(data: Mapping[str, Any], label: str) -> Optional[int]:
         int: The format version or None if it could not be found
     """
 
-    def read_version(item) -> Optional[str]:
+    def read_version(item) -> str | None:
         """try reading attribute from a particular item"""
         if hasattr(item, "attrs"):
             return read_version(item.attrs)
@@ -111,8 +111,8 @@ def _find_version(data: Mapping[str, Any], label: str) -> Optional[int]:
 
 
 def result_check_load_old_version(
-    path: Path, loc: str, *, model: Optional[ModelBase] = None
-) -> Optional[Result]:
+    path: Path, loc: str, *, model: ModelBase | None = None
+) -> Result | None:
     """check whether the resource can be loaded with an older version of the package
 
     Args:
@@ -132,13 +132,13 @@ def result_check_load_old_version(
     # check for compatibility
     fmt = guess_format(path)
     if fmt == "json":
-        with open(path, mode="r") as fp:
+        with open(path) as fp:
             format_version = _find_version(json.load(fp), label)
 
     elif fmt == "yaml":
         import yaml
 
-        with open(path, mode="r") as fp:
+        with open(path) as fp:
             format_version = _find_version(yaml.safe_load(fp), label)
 
     elif fmt == "hdf":

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/modelrunner/compatibility/version0.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 import json
 from pathlib import Path
-from typing import Any, Mapping, Optional
+from typing import Any, Mapping
 
 import numpy as np
 
@@ -32,7 +32,7 @@ def read_hdf_data(node):
 
 
 def _Result_from_simple_objects(
-    content: Mapping[str, Any], model: Optional[ModelBase] = None
+    content: Mapping[str, Any], model: ModelBase | None = None
 ) -> Result:
     """read result from simple object (like loaded from a JSON file) using version 0
 
@@ -54,7 +54,7 @@ def _Result_from_simple_objects(
     )
 
 
-def _Result_from_hdf(hdf_element, model: Optional[ModelBase] = None) -> Result:
+def _Result_from_hdf(hdf_element, model: ModelBase | None = None) -> Result:
     """old reader for backward compatible reading"""
     model_data = {key: json.loads(value) for key, value in hdf_element.attrs.items()}
     if "result" in hdf_element:
@@ -80,14 +80,14 @@ def result_from_file_v0(path: Path, **kwargs) -> Result:
     """
     fmt = guess_format(path)
     if fmt == "json":
-        with open(path, mode="r") as fp:
+        with open(path) as fp:
             content = json.load(fp)
         return _Result_from_simple_objects(content, **kwargs)
 
     elif fmt == "yaml":
         import yaml
 
-        with open(path, mode="r") as fp:
+        with open(path) as fp:
             content = yaml.safe_load(fp)
         return _Result_from_simple_objects(content, **kwargs)
 

{py-modelrunner-0.14.0 → py-modelrunner-0.16.0}/modelrunner/compatibility/version1.py

@@ -10,7 +10,7 @@ import json
 import warnings
 from abc import ABCMeta
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Optional, Type, TypeVar, Union
+from typing import TYPE_CHECKING, Any, TypeVar, Union
 
 import numpy as np
 
@@ -55,10 +55,10 @@ class StateBase(metaclass=ABCMeta):
     _state_format_version = 2
     """int: number indicating the version of the file format"""
 
-    _state_classes: Dict[str, Type[StateBase]] = {}
+    _state_classes: dict[str, type[StateBase]] = {}
     """dict: class-level list of all subclasses of StateBase"""
 
-    def _state_init(self, attributes: Dict[str, Any], data=NoData) -> None:
+    def _state_init(self, attributes: dict[str, Any], data=NoData) -> None:
         """initialize the state with attributes and (optionally) data
 
         Args:
@@ -71,7 +71,7 @@ class StateBase(metaclass=ABCMeta):
             self._state_attributes = attributes
 
     @classmethod
-    def from_data(cls: Type[TState], attributes: Dict[str, Any], data=NoData) -> TState:
+    def from_data(cls: type[TState], attributes: dict[str, Any], data=NoData) -> TState:
         """create instance of any state class from attributes and data
 
         Args:
@@ -84,7 +84,8 @@ class StateBase(metaclass=ABCMeta):
         # copy attributes since they are modified in this function
         attributes = attributes.copy()
         cls_name = attributes["__class__"]
-        assert cls_name == cls.__name__
+        if cls_name != cls.__name__:
+            raise RuntimeError(f"Expected `{cls.__name__}` but data had `{cls_name}`")
 
         format_version = attributes.pop("__version__", 0)
         if format_version != 1:
@@ -97,7 +98,7 @@ class StateBase(metaclass=ABCMeta):
         return obj
 
     @classmethod
-    def _from_simple_objects(cls, content: Dict[str, Any]) -> StateBase:
+    def _from_simple_objects(cls, content: dict[str, Any]) -> StateBase:
         """create state from text data
 
         Args:
@@ -115,7 +116,7 @@ class StateBase(metaclass=ABCMeta):
             return ArrayCollectionState.from_data(content["attributes"], col_data)
 
         elif cls_name == "DictState":
-            dict_data: Dict[str, StateBase] = {}
+            dict_data: dict[str, StateBase] = {}
             for label, substate in content["data"].items():
                 dict_data[label] = cls._from_simple_objects(substate)
 
@@ -128,7 +129,7 @@ class StateBase(metaclass=ABCMeta):
             raise TypeError(f"Do not know how to load `{cls_name}`")
 
     @classmethod
-    def _from_zarr(cls, zarr_element: "zarrElement", *, index=...) -> StateBase:
+    def _from_zarr(cls, zarr_element: zarrElement, *, index=...) -> StateBase:
         """create instance of correct subclass from data stored in zarr"""
         # determine the class that knows how to read this data
         cls_name = zarr_element.attrs["__class__"]
@@ -163,24 +164,20 @@ class StateBase(metaclass=ABCMeta):
             raise TypeError(f"Do not know how to load `{cls_name}`")
 
 
-class ArrayState(StateBase):
-    ...
+class ArrayState(StateBase): ...
 
 
-class ArrayCollectionState(StateBase):
-    ...
+class ArrayCollectionState(StateBase): ...
 
 
-class DictState(StateBase):
-    ...
+class DictState(StateBase): ...
 
 
-class ObjectState(StateBase):
-    ...
+class ObjectState(StateBase): ...
 
 
 def _Result_from_simple_objects(
-    content: Dict[str, Any], model: Optional[ModelBase] = None
+    content: dict[str, Any], model: ModelBase | None = None
 ) -> Result:
     """read result from simple object (like loaded from a JSON file) using version 1
 
@@ -202,7 +199,7 @@ def _Result_from_simple_objects(
 
 
 def _Result_from_zarr(
-    zarr_element: "zarrElement", *, index=..., model: Optional[ModelBase] = None
+    zarr_element: zarrElement, *, index=..., model: ModelBase | None = None
 ) -> Result:
     """create result from data stored in zarr"""
     attributes = {key: json.loads(value) for key, value in zarr_element.attrs.items()}
@@ -236,14 +233,14 @@ def result_from_file_v1(store: Path, *, label: str = "data", **kwargs) -> Result
     """
     fmt = guess_format(store)
     if fmt == "json":
-        with open(store, mode="r") as fp:
+        with open(store) as fp:
             content = json.load(fp)
         return _Result_from_simple_objects(content, **kwargs)
 
     elif fmt == "yaml":
         import yaml
 
-        with open(store, mode="r") as fp:
+        with open(store) as fp:
             content = yaml.safe_load(fp)
         return _Result_from_simple_objects(content, **kwargs)