py-modelrunner 0.17.0__tar.gz → 0.18.0__tar.gz

{py-modelrunner-0.17.0 → py_modelrunner-0.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py-modelrunner
-Version: 0.17.0
+Version: 0.18.0
 Summary: Python classes for organizing (HPC) simulations
 Author-email: David Zwicker <david.zwicker@ds.mpg.de>
 License: MIT
@@ -226,7 +226,7 @@ def multiply(a=1, b=2):
     return a * b
 
 if __name__ == "__main__":
-    submit_job(__file__, parameters={'a': 2}, output="data.hdf5", method="foreground")
+    submit_job(__file__, parameters={"a": 2}, output="data.hdf5", method="foreground")
 ```
 Here, the `output` argument specifies a file to which the results are written, while
 `method` chooses how the script is submitted.
@@ -243,7 +243,7 @@ def multiply(a=1, b=2):
 
 if __name__ == "__main__":
     for a in range(5):
-        submit_job(__file__, parameters={'a': a}, output=f"data_{a}.hdf5", method="foreground")
+        submit_job(__file__, parameters={"a": a}, output=f"data_{a}.hdf5", method="foreground")
 ```
 
 Note that the safe-guard `if __name__ == "__main__"` is absolutely crucial to ensure that
@@ -261,7 +261,7 @@ def multiply(a=1, b=2):
     return a * b
 
 if __name__ == "__main__":
-    submit_jobs(__file__, parameters={'a': range(5)}, output_folder="data", method="foreground")
+    submit_jobs(__file__, parameters={"a": range(5)}, output_folder="data", method="foreground")
 ```
 
 Finally, the packages also offers a method to submit a model script to the cluster using
@@ -301,7 +301,7 @@ For instance, the data from the multiple jobs ran above can be collected using
 from modelrunner import ResultCollection
 
 results = ResultCollection.from_folder(".", pattern="data_*.hdf5")
-print(results.dataframe)
+print(results.to_dataframe())
 ```
 
 This example should print all results using a pandas Dataframe, where each row

{py-modelrunner-0.17.0 → py_modelrunner-0.18.0}/README.md

@@ -184,7 +184,7 @@ def multiply(a=1, b=2):
     return a * b
 
 if __name__ == "__main__":
-    submit_job(__file__, parameters={'a': 2}, output="data.hdf5", method="foreground")
+    submit_job(__file__, parameters={"a": 2}, output="data.hdf5", method="foreground")
 ```
 Here, the `output` argument specifies a file to which the results are written, while
 `method` chooses how the script is submitted.
@@ -201,7 +201,7 @@ def multiply(a=1, b=2):
 
 if __name__ == "__main__":
     for a in range(5):
-        submit_job(__file__, parameters={'a': a}, output=f"data_{a}.hdf5", method="foreground")
+        submit_job(__file__, parameters={"a": a}, output=f"data_{a}.hdf5", method="foreground")
 ```
 
 Note that the safe-guard `if __name__ == "__main__"` is absolutely crucial to ensure that
@@ -219,7 +219,7 @@ def multiply(a=1, b=2):
     return a * b
 
 if __name__ == "__main__":
-    submit_jobs(__file__, parameters={'a': range(5)}, output_folder="data", method="foreground")
+    submit_jobs(__file__, parameters={"a": range(5)}, output_folder="data", method="foreground")
 ```
 
 Finally, the packages also offers a method to submit a model script to the cluster using
@@ -259,7 +259,7 @@ For instance, the data from the multiple jobs ran above can be collected using
 from modelrunner import ResultCollection
 
 results = ResultCollection.from_folder(".", pattern="data_*.hdf5")
-print(results.dataframe)
+print(results.to_dataframe())
 ```
 
 This example should print all results using a pandas Dataframe, where each row

{py-modelrunner-0.17.0 → py_modelrunner-0.18.0}/docs/source/conf.py

@@ -240,7 +240,6 @@ napoleon_custom_sections = None
 intersphinx_mapping = {
     "h5py": ("https://docs.h5py.org/en/latest", None),
     "matplotlib": ("https://matplotlib.org/stable", None),
-    "modelrunner": ("https://py-modelrunner.readthedocs.io/en/latest", None),
     "napari": ("https://napari.org/", None),
     "numba": ("https://numba.pydata.org/numba-doc/latest/", None),
     "numpy": ("https://numpy.org/doc/stable", None),

py_modelrunner-0.18.0/docs/source/sphinx_simplify_typehints.py

@@ -0,0 +1,74 @@
+"""
+Simple sphinx plug-in that simplifies  type information in function signatures
+"""
+
+import collections
+import re
+
+from modelrunner import Parameterized
+
+# simple (literal) replacement rules
+REPLACEMENTS = collections.OrderedDict(
+    [
+        # Remove some unnecessary information in favor of a more compact style
+        ("Dict[KT, VT]", "dict"),
+        ("Dict[str, Any]", "dict"),
+        ("Optional[str]", "str"),
+        ("Optional[float]", "float"),
+        ("Optional[int]", "int"),
+        ("Optional[dict]", "dict"),
+        ("Optional[Dict[str, Any]]", "dict"),
+    ]
+)
+
+
+# replacement rules based on regular expressions
+REPLACEMENTS_REGEX = {
+    # remove full package path and only leave the module/class identifier
+    r"modelrunner\.(\w+\.)*": "",
+}
+
+
+def process_signature(
+    app, what: str, name: str, obj, options, signature, return_annotation
+):
+    """Process signature by applying replacement rules"""
+    if signature is not None:
+        for key, value in REPLACEMENTS.items():
+            signature = signature.replace(key, value)
+        for key, value in REPLACEMENTS_REGEX.items():
+            signature = re.sub(key, value, signature)
+    return signature, return_annotation
+
+
+def sphinx_display_parameters(app, what, name, obj, options, lines):
+    """helper function to display parameters in sphinx documentation
+
+    Example:
+        This function should be connected to the 'autodoc-process-docstring'
+        event like so:
+
+            app.connect('autodoc-process-docstring', sphinx_display_parameters)
+    """
+    if what == "class" and issubclass(obj, Parameterized):
+        if any(":param parameters:" in line for line in lines):
+            # parse parameters
+            parameters = obj.get_parameters(sort=False)
+            if parameters:
+                lines.append(".. admonition::")
+                lines.append(f"   Parameters of {obj.__name__}:")
+                lines.append("   ")
+                for p in parameters.values():
+                    lines.append(f"   {p.name}")
+                    text = p.description.splitlines()
+                    text.append(f"(Default value: :code:`{p.default_value!r}`)")
+                    text = ["     " + t for t in text]
+                    lines.extend(text)
+                    lines.append("")
+                lines.append("")
+
+
+def setup(app):
+    """set up hooks for this sphinx plugin"""
+    app.connect("autodoc-process-signature", process_signature)
+    app.connect("autodoc-process-docstring", sphinx_display_parameters)

{py-modelrunner-0.17.0 → py_modelrunner-0.18.0}/docs/source/user_manual.rst

@@ -12,9 +12,9 @@ Hierarchical input/output using :mod:`~modelrunner.storage`:
     Storages are conveniently created by :func:`~modelrunner.storage.tools.open_storage`,
     which returns a storage object that offers a dict-like interface.
 
-Defining parameters using :mod:`~modelrunner.parameters`:
-    The module provides the :class:`~modelrunner.parameters.Parameter` class, describing a single parameter. 
-    This can be used together with the mixin :class:`~modelrunner.parameters.Parametrized`,
+Defining parameters using :mod:`~modelrunner.model.parameters`:
+    The module provides the :class:`~modelrunner.model.parameters.Parameter` class, describing a single parameter. 
+    This can be used together with the mixin :class:`~modelrunner.model.parameters.Parameterized`,
     which allows to equip classes with default parameters and some convenience methods.
 
 Defining models using :mod:`~modelrunner.model`:

{py-modelrunner-0.17.0 → py_modelrunner-0.18.0}/examples/model_storage_output.py

@@ -23,5 +23,5 @@ with tempfile.NamedTemporaryFile(suffix=".yaml") as fp:
 
     # read the file and check whether all the data is there
     with open_storage(fp.name) as storage:
-        print("Stored data:", storage["data"])
-        print("Model result:", storage["result"].result)
+        print("Stored data:", storage["storage/data"])
+        print("Model result:", storage["result"])

{py-modelrunner-0.17.0 → py_modelrunner-0.18.0}/modelrunner/__init__.py

@@ -18,15 +18,6 @@ except ImportError:
     del PackageNotFoundError, version  # clean name space
 
 
-from .model import (
-    ModelBase,
-    make_model,
-    make_model_class,
-    run_function_with_cmd_args,
-    run_script,
-    set_default,
-)
-from .parameters import HideParameter, Parameter, Parameterized
-from .results import Result, ResultCollection
-from .run import submit_job, submit_jobs
+from .model import *
+from .run import *
 from .storage import open_storage

{py-modelrunner-0.17.0 → py_modelrunner-0.18.0}/modelrunner/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.17.0'
-__version_tuple__ = version_tuple = (0, 17, 0)
+__version__ = version = '0.18.0'
+__version_tuple__ = version_tuple = (0, 18, 0)

{py-modelrunner-0.17.0 → py_modelrunner-0.18.0}/modelrunner/config.py

@@ -16,7 +16,7 @@ import contextlib
 from pathlib import Path
 from typing import Any, Sequence
 
-from .parameters import DeprecatedParameter, Parameter
+from .model.parameters import DeprecatedParameter, Parameter
 
 
 class Config(collections.UserDict):

py_modelrunner-0.18.0/modelrunner/model/__init__.py

@@ -0,0 +1,19 @@
+"""
+Defines models that handle a simulation and its inputs
+
+.. autosummary::
+   :nosignatures:
+
+   ~modelrunner.model.base.ModelBase
+   ~modelrunner.model.factory.make_model
+   ~modelrunner.model.factory.make_model_class
+   ~modelrunner.model.factory.set_default
+   ~modelrunner.model.parameters.Parameter
+   ~modelrunner.model.parameters.Parameterized
+
+.. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
+"""
+
+from .base import ModelBase
+from .factory import cleared_default_model, make_model, make_model_class, set_default
+from .parameters import HideParameter, Parameter, Parameterized

py_modelrunner-0.18.0/modelrunner/model/base.py

@@ -0,0 +1,247 @@
+"""
+Base class describing a model
+
+.. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+from abc import ABCMeta, abstractmethod
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Sequence
+
+from ..storage import ModeType, StorageGroup, open_storage
+from .parameters import DeprecatedParameter, HideParameter, Parameterized
+
+if TYPE_CHECKING:
+    from ..run.results import Result  # @UnusedImport
+
+
+class ModelBase(Parameterized, metaclass=ABCMeta):
+    """base class for describing models"""
+
+    name: str | None = None
+    """str: the name of the model"""
+    description: str | None = None
+    """str: a longer description of the model"""
+
+    def __init__(
+        self,
+        parameters: dict[str, Any] | None = None,
+        output: str | None = None,
+        *,
+        mode: ModeType = "insert",
+        strict: bool = False,
+    ):
+        """initialize the parameters of the object
+
+        Args:
+            parameters (dict):
+                A dictionary of parameters to change the defaults of this model. The
+                allowed parameters can be obtained from
+                :meth:`~Parameterized.get_parameters` or displayed by calling
+                :meth:`~Parameterized.show_parameters`.
+            output (str):
+                Path where the output file will be written. The output will be written
+                using :mod:`~modelrunner.storage` and might contain two groups: `result`
+                to which the final result of the model is written, and `data`, which
+                can contain extra information that is written using
+                :attr:`~ModelBase.storage`.
+            mode (str or :class:`~modelrunner.storage.access_modes.ModeType`):
+                The file mode with which the storage is accessed, which determines the
+                allowed operations. Common options are "read", "full", "append", and
+                "truncate".
+            strict (bool):
+                Flag indicating whether parameters are strictly interpreted. If `True`,
+                only parameters listed in `parameters_default` can be set and their type
+                will be enforced.
+        """
+        super().__init__(parameters, strict=strict)
+        self.output = output  # TODO: also allow already opened storages
+        self.mode = mode
+        self._storage: open_storage | None = None
+        self._logger = logging.getLogger(self.__class__.__name__)
+
+    @property
+    def storage(self) -> StorageGroup:
+        """:class:`StorageGroup`: Storage to which data can be written"""
+        if self._storage is None:
+            if self.output is None:
+                raise RuntimeError("Output file needs to be specified")
+            self._storage = open_storage(self.output, mode=self.mode)
+        return self._storage
+
+    def close(self) -> None:
+        """close any opened storages"""
+        if self._storage is not None:
+            self._storage.close()
+        self._storage = None
+
+    @abstractmethod
+    def __call__(self):
+        """main method calculating the result. Needs to be specified by sub-class"""
+
+    def get_result(self, data: Any = None) -> Result:
+        """get the result as a :class:`~model.Result` object
+
+        Args:
+            data:
+                The result data. If omitted, the model is run to obtain results
+
+        Returns:
+            :class:`Result`: The result after the model is run
+        """
+        from ..run.results import Result  # @Reimport
+
+        if data is None:
+            data = self()
+
+        info = {"time": datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
+        return Result(self, data, info=info)
+
+    def write_result(self, result: Result | None = None) -> None:
+        """write the result to the output file
+
+        Args:
+            result:
+                The result data. If omitted, the model is run to obtain results
+        """
+        from ..run.results import Result  # @Reimport
+
+        if self.output is None:
+            raise RuntimeError("Output file needs to be specified")
+
+        if result is None:
+            result = self.get_result()
+        elif not isinstance(result, Result):
+            raise TypeError(f"result has type {result.__class__} instead of `Result`")
+
+        if self._storage is not None:
+            # reuse the opened storage
+            result.to_file(self._storage)
+        else:
+            result.to_file(self.output, mode=self.mode)
+
+    @classmethod
+    def _prepare_argparser(cls, name: str | None = None) -> argparse.ArgumentParser:
+        """create argument parser for setting parameters of this model
+
+        Args:
+            name (str):
+                Name of the program, which will be shown in the command line help
+
+        Returns:
+            :class:`~argparse.ArgumentParser`
+        """
+        parser = argparse.ArgumentParser(
+            prog=name,
+            description=cls.description,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        )
+
+        group = parser.add_argument_group()
+        seen = set()
+        # iterate over all parent classes
+        for cls1 in cls.__mro__:
+            if hasattr(cls1, "parameters_default"):
+                # add all parameters of this class
+                for p in cls1.parameters_default:
+                    if p.name not in seen:
+                        if not isinstance(p, (HideParameter, DeprecatedParameter)):
+                            p._argparser_add(group)
+                        seen.add(p.name)
+
+        # add special parameters
+        parser.add_argument(
+            "--json",
+            metavar="JSON",
+            help="JSON-encoded parameter values. Overwrites other parameters.",
+        )
+
+        return parser
+
+    @classmethod
+    def from_command_line(
+        cls, args: Sequence[str] | None = None, name: str | None = None
+    ) -> ModelBase:
+        """create model from command line parameters
+
+        Args:
+            args (list):
+                Sequence of strings corresponding to the command line arguments
+            name (str):
+                Name of the program, which will be shown in the command line help
+
+        Returns:
+            :class:`ModelBase`: An instance of this model with appropriate parameters
+        """
+        if args is None:
+            args = []
+
+        # read the command line arguments
+        parser = cls._prepare_argparser(name)
+
+        # add special parameters to determine the output file
+        parser.add_argument(
+            "-o",
+            "--output",
+            metavar="PATH",
+            help="Path to output file. If omitted, no output file is created.",
+        )
+
+        parameters = vars(parser.parse_args(args))
+        output = parameters.pop("output")
+        parameters_json = parameters.pop("json")
+
+        # update parameters with data from the json argument
+        if parameters_json:
+            parameters.update(json.loads(parameters_json))
+
+        # create the model
+        return cls(parameters, output=output)
+
+    @classmethod
+    def run_from_command_line(
+        cls, args: Sequence[str] | None = None, name: str | None = None
+    ) -> Result:
+        """run model using command line parameters
+
+        Args:
+            args (list):
+                Sequence of strings corresponding to the command line arguments
+            name (str):
+                Name of the program, which will be shown in the command line help
+
+        Returns:
+            :class:`Result`: The result of running the model
+        """
+        # create model from command line parameters
+        mdl = cls.from_command_line(args, name)
+
+        # run the model
+        result = mdl.get_result()
+        if mdl.output:
+            # write the results to a file
+            mdl.write_result(result=result)
+        # else:
+        #     # display the results on stdout
+        #     storage = MemoryStorage()
+        #     result.to_file(storage)
+
+        # close the output file
+        mdl.close()
+
+        return result
+
+    @property
+    def _state_attributes(self) -> dict[str, Any]:
+        """dict: information about the element state, which does not change in time"""
+        return {
+            "class": self.__class__.__name__,
+            "name": self.name,
+            "description": self.description,
+            "parameters": self.parameters,
+        }

py_modelrunner-0.18.0/modelrunner/model/factory.py

@@ -0,0 +1,173 @@
+"""
+Functions for creating models and model classes from other input
+
+.. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
+"""
+
+from __future__ import annotations
+
+import functools
+import inspect
+from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, get_args, get_origin
+
+from ..storage import ModeType
+from .base import ModelBase
+from .parameters import NoValue, Parameter
+
+if TYPE_CHECKING:
+    from ..run.results import Result  # @UnusedImport
+
+_DEFAULT_MODEL: Callable | ModelBase | None = None
+"""stores the default model that will be used automatically"""
+
+
+TModel = TypeVar("TModel", Callable, ModelBase, None)
+
+
+def set_default(func_or_model: TModel) -> TModel:
+    """sets the function or model as the default model
+
+    The last model that received this flag will be run automatically. This only affects
+    the behavior when the script is run using `modelrunner` from the command line, e.g.,
+    using :code:`python -m modelrunner script.py`.
+
+    Args:
+        func_or_model (callabel or :class:`ModelBase`, optional):
+            The function or model that should be called when the script is run.
+
+    Returns:
+        `func_or_model`, so the function can be used as a decorator
+    """
+    global _DEFAULT_MODEL
+    _DEFAULT_MODEL = func_or_model
+    return func_or_model
+
+
+TFunc = TypeVar("TFunc", bound=Any)
+
+
+def cleared_default_model(func: TFunc) -> TFunc:
+    """run the function with a cleared _DEFAULT_MODEL and restore it afterwards"""
+
+    @functools.wraps(func)
+    def inner(*args, **kwargs):
+        global _DEFAULT_MODEL
+        _old_model = _DEFAULT_MODEL
+        _DEFAULT_MODEL = None
+        try:
+            return func(*args, **kwargs)
+        finally:
+            _DEFAULT_MODEL = _old_model
+
+    return inner  # type: ignore
+
+
+def make_model_class(func: Callable, *, default: bool = False) -> type[ModelBase]:
+    """create a model from a function by interpreting its signature
+
+    Args:
+        func (callable):
+            The function that will be turned into a Model
+        default (bool):
+            If True, set this model as the default one for the current script
+
+    Returns:
+        :class:`ModelBase`: A subclass of ModelBase, which encompasses `func`
+    """
+    # determine the parameters of the function
+    provide_storage = False
+    parameters_default = []
+    for name, param in inspect.signature(func).parameters.items():
+        if name == "storage":
+            # treat this parameter specially and provide access to a storage object
+            provide_storage = True
+        else:
+            # all remaining parameters are treated as model parameters
+            if param.annotation is param.empty:
+                cls = object
+                choices = None
+            elif get_origin(param.annotation) is Literal:
+                cls = object
+                choices = get_args(param.annotation)
+            else:
+                cls = param.annotation
+                choices = None
+            if param.default is param.empty:
+                default_value = NoValue
+            else:
+                default_value = param.default
+
+            parameter = Parameter(
+                name, default_value=default_value, cls=cls, choices=choices
+            )
+            parameters_default.append(parameter)
+
+    def __call__(self, *args, **kwargs):
+        """call the function preserving the original signature"""
+        parameters = {}
+        for i, (name, value) in enumerate(self.parameters.items()):
+            if len(args) > i:
+                if name in kwargs:
+                    raise ValueError(f"{name} also given as positional argument")
+                param_value = args[i]
+            elif name in kwargs:
+                param_value = kwargs[name]
+            else:
+                param_value = value
+
+            if param_value is NoValue:
+                raise TypeError(f"Model missing required argument: '{name}'")
+            parameters[name] = param_value
+
+        if provide_storage:
+            if "storage" in self.storage:
+                self._logger.info("Open storage group `storage`")
+                parameters["storage"] = self.storage.open_group("storage")
+            else:
+                self._logger.info("Create storage group `storage`")
+                parameters["storage"] = self.storage.create_group("storage")
+
+        return func(**parameters)
+
+    args = {
+        "name": func.__name__,
+        "description": func.__doc__,
+        "parameters_default": parameters_default,
+        "__doc__": func.__doc__,
+        "__call__": __call__,
+    }
+    newclass = type(func.__name__, (ModelBase,), args)
+    if default:
+        set_default(newclass)
+    return newclass
+
+
+def make_model(
+    func: Callable,
+    parameters: dict[str, Any] | None = None,
+    output: str | None = None,
+    *,
+    mode: ModeType = "insert",
+    default: bool = False,
+) -> ModelBase:
+    """create model from a function and a dictionary of parameters
+
+    Args:
+        func (callable):
+            The function that will be turned into a Model
+        parameters (dict):
+            Paramter values with which the model is initialized
+        output (str):
+            Path where the output file will be written.
+        mode (str or :class:`~modelrunner.storage.access_modes.ModeType`):
+            The file mode with which the storage is accessed, which determines the
+            allowed operations. Common options are "read", "full", "append", and
+            "truncate".
+        default (bool):
+            If True, set this model as the default one for the current script
+
+    Returns:
+        :class:`ModelBase`: An instance of a subclass of ModelBase encompassing `func`
+    """
+    model_class = make_model_class(func, default=default)
+    return model_class(parameters, output=output, mode=mode)

{py-modelrunner-0.17.0/modelrunner → py_modelrunner-0.18.0/modelrunner/model}/parameters.py

@@ -25,7 +25,7 @@ from typing import Any, Callable, Container, Dict, Iterator, List, Optional, Uni
 
 import numpy as np
 
-from .utils import hybridmethod, import_class
+from ..utils import hybridmethod, import_class
 
 
 class NoValueType: