py-modelrunner 0.18.0__tar.gz → 0.18.1__tar.gz

{py_modelrunner-0.18.0 → py_modelrunner-0.18.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py-modelrunner
-Version: 0.18.0
+Version: 0.18.1
 Summary: Python classes for organizing (HPC) simulations
 Author-email: David Zwicker <david.zwicker@ds.mpg.de>
 License: MIT

{py_modelrunner-0.18.0 → py_modelrunner-0.18.1}/docs/source/user_manual.rst

@@ -20,33 +20,33 @@ Defining parameters using :mod:`~modelrunner.model.parameters`:
 Defining models using :mod:`~modelrunner.model`:
     Models are augmented functions, which define input, calculations, and output.
     Models can be conveniently created by decorating a function or by subclassing
-    :class:`~modelrunner.model.ModelBase`, which is built on the parameter classes.
+    :class:`~modelrunner.model.base.ModelBase`, which is built on the parameter classes.
 
-Model results are captured by :mod:`~modelrunner.results`:
-    Results are returned as the special :class:`~modelrunner.results.Result`, which
+Model results are captured by :mod:`~modelrunner.run.results`:
+    Results are returned as the special :class:`~modelrunner.run.results.Result`, which
     keeps track of the input parameters and the data calculated by the model.
-    :class:`~modelrunner.results.ResultCollection` describes a collections of the same
+    :class:`~modelrunner.run.results.ResultCollection` describes a collections of the same
     model evoked with different parameter values.
 
 Submitting models to HPC using :mod:`~modelrunner.run`:
     A single model can be submitted to a compute node using :func:`~modelrunner.run.job.submit_job`,
     e.g., to run the computation on a high performance compute cluster.
     A parameter study using multiple jobs can be conveniently submitted using :func:`~modelrunner.run.job.submit_jobs`.
-    The results written to one directory can then be conveniently analyzed using :meth:`~modelrunner.results.ResultCollection.from_folder`.
+    The results written to one directory can then be conveniently analyzed using :func:`~modelrunner.run.results.ResultCollection.from_folder`.
 
 
 Design philosophy
 -----------------
 
-The main requirements for the storage classes were
+The main requirements for the package can be summarized as follows:
 
-- *Usability*: The user should not need to think about how data is stored in different files
-- *Flexibility*: We want a general interface to write data in multiple file formats (YAML, HDF, zarr, ...)
-- *Stability*: Future versions of the package should be able to read older files even when the internal definitions of file formats change
-- *Modularity*: Different parts of the package (like :mod:`~modelrunner.storage`, :mod:`~modelrunner.parameters`, and :mod:`~modelrunner.run`) should be rather independent of each other, so they can be used in isolation
-- *Extensibility*: Models should be easy to subclass to implement more complicated requirements (e.g., additional parameters)
-- *Self-explainability*: The files should in principle contain all information to reconstruct the data, even if the `py-modelrunner` package is no longer available.
-- *Efficiency*: The files should only store necessary information.
+- **Usability**: The user should not need to think about how data is stored in different files. The :class:`~modelrunner.run.results.Result` class should simply work.
+- **Flexibility**: :mod:`~modelrunner.storage` should provide a unified interface to write data in multiple file formats (JSON, YAML, HDF, zarr, ...)
+- **Stability**: Future versions of the package should be able to read older files even when the internal definitions of file formats change
+- **Modularity**: Different parts of the package (like :mod:`~modelrunner.storage`, :mod:`~modelrunner.model.parameters`, and :mod:`~modelrunner.run`) should be rather independent of each other, so they can be used in isolation
+- **Extensibility**: Models inherting from :class:`~modelrunner.model.base.ModelBase` should be easy to subclass to implement more complicated requirements (e.g., additional parameters)
+- **Self-explainability**: The files should in principle contain all information to reconstruct the data, even if the :mod:`modelrunner` package is no longer available.
+- **Efficiency**: The files should only store necessary information.
 
 The last point results in particular constraints if we want to store temporal simulation results.
 In most cases, there are are some data that are kept fixed for the simulation (describing physical parameters) and others that evolve with time.

{py_modelrunner-0.18.0 → py_modelrunner-0.18.1}/modelrunner/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.18.0'
-__version_tuple__ = version_tuple = (0, 18, 0)
+__version__ = version = '0.18.1'
+__version_tuple__ = version_tuple = (0, 18, 1)

{py_modelrunner-0.18.0 → py_modelrunner-0.18.1}/modelrunner/run/job.py

@@ -219,10 +219,13 @@ def submit_job(
     job_args = []
     if parameters is not None and len(parameters) > 0:
         if isinstance(parameters, dict):
-            parameters = json.dumps(parameters)
-        elif not isinstance(parameters, str):
+            parameters_json = json.dumps(parameters)
+        elif isinstance(parameters, str):
+            parameters_json = parameters
+        else:
             raise TypeError("Parameters need to be given as a string or a dict")
-        job_args.append(f"--json {escape_string(parameters)}")
+        job_args.append(f"--json {escape_string(parameters_json)}")
+        script_args["PARAMETERS"] = parameters  # allow using parameters in job script
 
     logger.debug("Job arguments: `%s`", job_args)
 

{py_modelrunner-0.18.0 → py_modelrunner-0.18.1}/modelrunner/run/results.py

@@ -55,7 +55,7 @@ class Result:
         The result is represented as a hierarchical structure when safed using the
         :mod:`~modelrunner.storage`. The actual result is stored in the `result` group,
         whereas the model information can be found in `_model` group. Additional
-        information is stored in the root attribute. Thus, the full :class:`Result` can
+        information is stored in the `storage` group. Thus, the full :class:`Result` can
         be read using :code:`storage[loc]`, where `loc` denotes the result location. If
         only the actual result is needed, :code:`storage[loc + "/result"]` can be read.
     """

{py_modelrunner-0.18.0 → py_modelrunner-0.18.1}/modelrunner/storage/backend/text_base.py

@@ -80,7 +80,7 @@ class TextStorageBase(MemoryStorage, metaclass=ABCMeta):
         elif self._modified:
             # The storage was modified, but it cannot be written to the file. This
             # should not happen, but it's better to throw an explicit error
-            raise AccessError("Cannot write to file")
+            raise AccessError("Cannot write modifications to file opened read-only")
 
     def close(self) -> None:
         """close the file and write the data to the file"""

{py_modelrunner-0.18.0 → py_modelrunner-0.18.1}/modelrunner/storage/utils.py

@@ -159,13 +159,20 @@ class _StorageRegistry:
     allowed_actions = set(ActionType.__args__)  # type: ignore
     """set: all actions that can be registered"""
 
-    _hooks: dict[type, dict[str, Callable]]
+    _hooks: dict[type, dict[str, tuple[Callable, bool]]]
     """dict: register for all defined hooks"""
 
     def __init__(self):
         self._hooks = defaultdict(dict)
 
-    def register(self, action: ActionType, cls: type, method_or_func: Callable) -> None:
+    def register(
+        self,
+        action: ActionType,
+        cls: type,
+        method_or_func: Callable,
+        *,
+        inherit: bool = True,
+    ) -> None:
         """register an action for the given class
 
         Example:
@@ -182,6 +189,9 @@ class _StorageRegistry:
                 The class this action is associated with
             method_or_func (callable):
                 The function/method that is called for the action
+            inherit (bool):
+                Determines whether child classes of `cls` inherit this action and will
+                be able to use it.
         """
         if action not in self.allowed_actions:
             raise ValueError(f"Unknown action `{action}` ")
@@ -192,9 +202,9 @@ class _StorageRegistry:
                 """helper function to call the classmethod"""
                 return method_or_func(cls, *args, **kwargs)
 
-            self._hooks[cls][action] = _call_classmethod
+            self._hooks[cls][action] = (_call_classmethod, inherit)
         elif callable(method_or_func):
-            self._hooks[cls][action] = method_or_func
+            self._hooks[cls][action] = (method_or_func, inherit)
         else:
             raise TypeError("`method_or_func` must be method or function")
 
@@ -214,7 +224,9 @@ class _StorageRegistry:
         classes = inspect.getmro(cls)[:-1]
         for c in classes:
             if c in self._hooks and action in self._hooks[c]:
-                return self._hooks[c][action]
+                func, inherit = self._hooks[c][action]
+                if inherit or c is cls:
+                    return func
 
         raise RuntimeError(f"No action `{action}` for `{cls.__name__}`")
 

{py_modelrunner-0.18.0 → py_modelrunner-0.18.1}/py_modelrunner.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py-modelrunner
-Version: 0.18.0
+Version: 0.18.1
 Summary: Python classes for organizing (HPC) simulations
 Author-email: David Zwicker <david.zwicker@ds.mpg.de>
 License: MIT

{py_modelrunner-0.18.0 → py_modelrunner-0.18.1}/py_modelrunner.egg-info/SOURCES.txt

@@ -92,6 +92,7 @@ scripts/run_tests.py
 scripts/tests_all.sh
 scripts/tests_codestyle.sh
 scripts/tests_coverage.sh
+scripts/tests_debug.sh
 scripts/tests_parallel.sh
 scripts/tests_run.sh
 scripts/tests_types.sh
@@ -197,11 +198,13 @@ tests/model/scripts/make_model_marked.py
 tests/run/test_job.py
 tests/run/test_results.py
 tests/run/test_run_main.py
+tests/run/scripts/custom.jinja
 tests/run/scripts/fail.py
 tests/run/scripts/function.py
 tests/run/scripts/print.py
 tests/run/scripts/script.py
 tests/storage/test_group.py
+tests/storage/test_register_action.py
 tests/storage/test_storage_generic.py
 tests/storage/test_storage_special.py
 tests/storage/test_trajectory.py

{py_modelrunner-0.18.0 → py_modelrunner-0.18.1}/scripts/run_tests.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+from __future__ import annotations
+
 import argparse
 import os
 import subprocess as sp
@@ -66,6 +68,7 @@ def run_unit_tests(
     coverage: bool = False,
     no_numba: bool = False,
     pattern: str = None,
+    pytest_args: list[str] = [],
 ) -> int:
     """run the unit tests
 
@@ -74,6 +77,9 @@ def run_unit_tests(
         coverage (bool): Whether to determine the test coverage
         no_numba (bool): Whether to disable numba jit compilation
         pattern (str): A pattern that determines which tests are ran
+        pytest_args (list of str):
+            Additional arguments forwarded to py.test. For instance ["--maxfail=1"]
+            fails tests early.
 
     Returns:
         int: The return code indicating success or failure
@@ -117,6 +123,8 @@ def run_unit_tests(
             ]
         )
 
+    args.extend(pytest_args)
+
     # specify the package to run
     args.append("tests")
 
@@ -191,6 +199,13 @@ def main():
         help="Write a report of the results",
     )
 
+    # set py.test arguments
+    group = parser.add_argument_group(
+        "py.test arguments",
+        description="Additional arguments separated by `--` are forward to py.test",
+    )
+    group.add_argument("pytest_args", nargs="*", help=argparse.SUPPRESS)
+
     # parse the command line arguments
     args = parser.parse_args()
     run_all = not (args.style or args.types or args.unit)
@@ -206,6 +221,7 @@ def main():
             parallel=args.parallel,
             no_numba=args.no_numba,
             pattern=args.pattern,
+            pytest_args=args.pytest_args,
         )
 
 

py_modelrunner-0.18.1/scripts/tests_debug.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+export PYTHONPATH=../py-modelrunner  # likely path of package, relative to current base path
+
+if [ ! -z $1 ] 
+then 
+    # test pattern was specified 
+    echo 'Run unittests with pattern '$1':'
+    ./run_tests.py --unit --pattern "$1" -- \
+        -o log_cli=true --log-cli-level=debug -vv
+else
+    # test pattern was not specified
+    echo 'Run all unittests:'
+    ./run_tests.py --unit -- \
+        -o log_cli=true --log-cli-level=debug -vv
+fi

py_modelrunner-0.18.1/tests/run/scripts/custom.jinja

@@ -0,0 +1,19 @@
+#!/bin/bash -l
+
+export PYTHONPATH={{ PACKAGE_PATH }}:$PYTHONPATH
+
+{% if CONFIG.num_threads is number %}
+# set the number of threads to use
+export MKL_NUM_THREADS={{ CONFIG.num_threads }}
+export NUMBA_NUM_THREADS={{ CONFIG.num_threads }}
+export NUMEXPR_NUM_THREADS={{ CONFIG.num_threads }}
+export OMP_NUM_THREADS={{ CONFIG.num_threads }}
+export OPENBLAS_NUM_THREADS={{ CONFIG.num_threads }}
+{% endif %}
+
+{% if OUTPUT_FOLDER is defined and OUTPUT_FOLDER %}
+mkdir -p {{ OUTPUT_FOLDER }}
+{% endif %}
+
+# Run the program
+{{ CONFIG.python_bin }} -m modelrunner {{ MODEL_FILE }} --a {{ PARAMETERS.a }}

{py_modelrunner-0.18.0 → py_modelrunner-0.18.1}/tests/run/test_job.py

@@ -57,8 +57,8 @@ def test_submit_job_stdout(tmp_path, method):
         overwrite_strategy="silent_overwrite",
     )
 
-    assert outs == "3.0\n"
     assert errs == ""
+    assert outs == "3.0\n"
     assert Result.from_file(output).result is None
 
 
@@ -69,8 +69,8 @@ def test_submit_job_no_output():
         method="foreground",
         overwrite_strategy="silent_overwrite",
     )
-    assert outs == "3.0\n"
     assert errs == ""
+    assert outs == "3.0\n"
 
 
 def test_submit_jobs(tmp_path):
@@ -137,3 +137,17 @@ def test_submit_job_no_modelrunner(tmp_path):
 
     assert run() == ("", "")
     assert run(a=1) == ('--json{"a": 1}', "")
+
+
+def test_submit_job_own_template(tmp_path):
+    """test the submit_job function with a custom template"""
+    outs, errs = submit_job(
+        SCRIPT_PATH / "print.py",
+        method="foreground",
+        parameters={"a": 5, "b": 10},  # b is not used by template
+        template=SCRIPT_PATH / "custom.jinja",
+        overwrite_strategy="silent_overwrite",
+    )
+
+    assert errs == ""
+    assert outs == "7.0\n"  # a + 2 (default value of b)

py_modelrunner-0.18.1/tests/storage/test_register_action.py

@@ -0,0 +1,155 @@
+"""
+.. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
+"""
+
+import pytest
+
+from helpers import STORAGE_EXT, STORAGE_OBJECTS, assert_data_equals
+from modelrunner.storage import StorageGroup, open_storage, storage_actions
+
+
+class A:
+
+    def __init__(self, value):
+        self.value = value
+        self.loaded = False
+
+    def save(self, storage, loc):
+        storage.write_object(loc, self.value, cls=self.__class__)
+
+    @classmethod
+    def load(cls, storage, loc):
+        obj = cls(storage.read_object(loc))
+        obj.loaded = True
+        return obj
+
+
+def save_A(storage, loc, obj):
+    obj.save(storage, loc)
+
+
+storage_actions.register("read_item", A, A.load, inherit=True)
+storage_actions.register("write_item", A, save_A, inherit=True)
+
+
+class A1(A): ...
+
+
+class A2(A):
+
+    @classmethod
+    def load(cls, storage, loc):
+        obj = cls(storage.read_object(loc))
+        obj.loaded = "yes"
+        return obj
+
+
+storage_actions.register("read_item", A2, A2.load, inherit=False)
+
+
+class B:
+
+    def __init__(self, a, b):
+        self.a = a
+        self.b = b
+        self.loaded = False
+
+    def save(self, storage, loc):
+        group = storage.create_group(loc, cls=self.__class__)
+        self.a.save(group, "a")
+        group.write_object("b", self.b)
+
+    @classmethod
+    def load(cls, storage, loc):
+        group = StorageGroup(storage, loc)
+        obj = cls(a=group["a"], b=group.read_object("b"))
+        obj.loaded = True
+        return obj
+
+
+storage_actions.register("read_item", B, B.load, inherit=False)
+storage_actions.register("write_item", B, lambda s, l, o: o.save(s, l), inherit=False)
+
+
+class B1(B): ...
+
+
+@pytest.mark.parametrize("ext", STORAGE_EXT)
+def test_register_class_action(ext, tmp_path):
+    """test loading custom classes"""
+
+    a = A({"a": 1})
+    assert not a.loaded
+
+    with open_storage(tmp_path / f"file{ext}", mode="truncate") as store:
+        store["a"] = a
+
+    with open_storage(tmp_path / f"file{ext}", mode="read") as store:
+        b = store["a"]
+
+    assert b.loaded
+    assert b.value == {"a": 1}
+
+
+@pytest.mark.parametrize("ext", STORAGE_EXT)
+def test_register_class_action_nested(tmp_path, ext):
+    """test loading custom nested classes"""
+
+    a = A({"a": 1})
+    b = B(a, "str")
+    assert not a.loaded and not b.loaded
+
+    with open_storage(tmp_path / f"file{ext}", mode="truncate") as store:
+        store["data"] = b
+
+    with open_storage(tmp_path / f"file{ext}", mode="read") as store:
+        res = store["data"]
+
+    assert res.loaded and res.a.loaded
+    assert res.a.value == {"a": 1}
+    assert res.b == "str"
+
+
+@pytest.mark.parametrize("ext", STORAGE_EXT)
+def test_register_class_action_subclass(ext, tmp_path):
+    """test loading custom classes and subclasses"""
+
+    a = A({"a": 1})
+    a1 = A1({"b": 2})  # has not defined hooks itself
+    a2 = A2({"c": 3})  # has own loading hook
+    assert not a.loaded and not a1.loaded and not a2.loaded
+
+    with open_storage(tmp_path / f"file{ext}", mode="truncate") as store:
+        store["a"] = a
+        store["a1"] = a1
+        store["a2"] = a2
+
+    with open_storage(tmp_path / f"file{ext}", mode="read") as store:
+        b = store["a"]
+        b1 = store["a1"]
+        b2 = store["a2"]
+
+    assert b.loaded and b1.loaded and b2.loaded == "yes"
+    assert b.value == {"a": 1} and b1.value == {"b": 2} and b2.value == {"c": 3}
+
+
+@pytest.mark.parametrize("ext", STORAGE_EXT)
+def test_register_class_action_no_inhert(ext, tmp_path):
+    """test loading custom classes and subclasses"""
+
+    a = B(A({"a": 1}), 2)  # does not use hooks that inherit
+    a1 = B1(A({"a": 2}), 4)  # has not defined hooks itself
+    assert not a.loaded and not a1.loaded
+
+    with open_storage(tmp_path / f"file{ext}", mode="truncate") as store:
+        store["b"] = a
+        store["b1"] = a1
+
+    with open_storage(tmp_path / f"file{ext}", mode="read") as store:
+        b = store["b"]
+        b1 = store["b1"]
+
+    assert b.loaded  # B uses the hook class
+    assert not b1.loaded  # B2 does not yse the hook classes
+    assert b.a.value == {"a": 1} and b.b == 2
+    assert b1.a.value == {"a": 2} and b1.b == 4

{py_modelrunner-0.18.0 → py_modelrunner-0.18.1}/tests/storage/test_storage_generic.py

@@ -6,7 +6,7 @@ import numpy as np
 import pytest
 
 from helpers import STORAGE_EXT, STORAGE_OBJECTS, assert_data_equals
-from modelrunner.storage import AccessError, StorageGroup, open_storage, storage_actions
+from modelrunner.storage import AccessError, open_storage
 
 OBJ = {"a": 1, "b": np.arange(5)}
 ARRAY_EXAMPLES = [
@@ -328,87 +328,3 @@ def test_storage_close(ext, tmp_path):
     assert reader["obj"] == 1
     reader.close()
     assert reader.closed
-
-
-class A:
-
-    def __init__(self, value):
-        self.value = value
-        self.loaded = False
-
-    def save(self, storage, loc):
-        storage.write_object(loc, self.value, cls=self.__class__)
-
-    @classmethod
-    def load(cls, storage, loc):
-        obj = cls(storage.read_object(loc))
-        obj.loaded = True
-        return obj
-
-
-def save_A(storage, loc, obj):
-    obj.save(storage, loc)
-
-
-storage_actions.register("read_item", A, A.load)
-storage_actions.register("write_item", A, save_A)
-
-
-class B:
-
-    def __init__(self, a, b):
-        self.a = a
-        self.b = b
-        self.loaded = False
-
-    def save(self, storage, loc):
-        group = storage.create_group(loc, cls=self.__class__)
-        self.a.save(group, "a")
-        group.write_object("b", self.b)
-
-    @classmethod
-    def load(cls, storage, loc):
-        group = StorageGroup(storage, loc)
-        obj = cls(a=group["a"], b=group.read_object("b"))
-        obj.loaded = True
-        return obj
-
-
-storage_actions.register("read_item", B, B.load)
-storage_actions.register("write_item", B, lambda s, l, o: o.save(s, l))
-
-
-@pytest.mark.parametrize("ext", STORAGE_EXT)
-def test_register_class_action(ext, tmp_path):
-    """test loading custom classes"""
-
-    a = A({"a": 1})
-    assert not a.loaded
-
-    with open_storage(tmp_path / f"file{ext}", mode="truncate") as store:
-        store["a"] = a
-
-    with open_storage(tmp_path / f"file{ext}", mode="read") as store:
-        b = store["a"]
-
-    assert b.loaded
-    assert b.value == {"a": 1}
-
-
-@pytest.mark.parametrize("ext", STORAGE_EXT)
-def test_register_class_action_nested(tmp_path, ext):
-    """test loading custom nested classes"""
-
-    a = A({"a": 1})
-    b = B(a, "str")
-    assert not a.loaded and not b.loaded
-
-    with open_storage(tmp_path / f"file{ext}", mode="truncate") as store:
-        store["data"] = b
-
-    with open_storage(tmp_path / f"file{ext}", mode="read") as store:
-        res = store["data"]
-
-    assert res.loaded and res.a.loaded
-    assert res.a.value == {"a": 1}
-    assert res.b == "str"