py-modelrunner 0.13.0__tar.gz → 0.14.0__tar.gz

{py-modelrunner-0.13.0 → py-modelrunner-0.14.0}/.gitignore

@@ -122,6 +122,12 @@ venv.bak/
 # Rope project settings
 .ropeproject
 
+# VSCode project settings
+.vscode
+
+# Eclipse project settings
+.settings
+
 # mkdocs documentation
 /site
 

{py-modelrunner-0.13.0 → py-modelrunner-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py-modelrunner
-Version: 0.13.0
+Version: 0.14.0
 Summary: Python classes for organizing (HPC) simulations
 Author-email: David Zwicker <david.zwicker@ds.mpg.de>
 License: MIT
@@ -18,9 +18,26 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.8
+Requires-Python: <3.12,>=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: jinja2>=3
+Requires-Dist: numpy>=1.18.0
+Requires-Dist: tqdm>=4.45
+Requires-Dist: numcodecs>=0.10
+Provides-Extra: all
+Requires-Dist: h5py>=3.5; extra == "all"
+Requires-Dist: pandas>=1.3; extra == "all"
+Requires-Dist: PyYAML>=5; extra == "all"
+Requires-Dist: zarr>=2; extra == "all"
+Provides-Extra: test
+Requires-Dist: black>=19; extra == "test"
+Requires-Dist: isort>=5.1; extra == "test"
+Requires-Dist: pytest>=5.4; extra == "test"
+Requires-Dist: pytest-cov>=2.8; extra == "test"
+Requires-Dist: pytest-xdist>=1.30; extra == "test"
+Requires-Dist: py-pde>=0.33; extra == "test"
+Requires-Dist: mypy>=0.770; extra == "test"
 
 # py-modelrunner
 

{py-modelrunner-0.13.0 → py-modelrunner-0.14.0}/examples/io_hdf.py

@@ -22,4 +22,4 @@ if __name__ == "__main__":
 
     # write result from file
     read = Result.from_file("test.hdf")
-    print(read.parameters, "–– start + [0..length-1] =", read.state)
+    print(read.parameters, "–– start + [0..length-1] =", read.data)

{py-modelrunner-0.13.0 → py-modelrunner-0.14.0}/examples/io_json.py

@@ -18,4 +18,4 @@ if __name__ == "__main__":
     result.to_file("test.json")
 
     read = Result.from_file("test.json")
-    print(read.parameters, "–– a * b =", read.state)
+    print(read.parameters, "–– a * b =", read.data)

{py-modelrunner-0.13.0 → py-modelrunner-0.14.0}/examples/io_yaml.py

@@ -18,4 +18,4 @@ if __name__ == "__main__":
     result.to_file("test.yaml")
 
     read = Result.from_file("test.yaml")
-    print(read.parameters, "–– a * b =", read.state)
+    print(read.parameters, "–– a * b =", read.data)

{py-modelrunner-0.13.0 → py-modelrunner-0.14.0}/examples/model_class.py

@@ -5,6 +5,8 @@ This example shows defining a custom model class by subclassing.
 .. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
 """
 
+import numpy as np
+
 from modelrunner import ModelBase
 
 
@@ -12,10 +14,12 @@ class MyModel(ModelBase):
     parameters_default = {"a": 1, "b": 2}
 
     def __call__(self):
+        self.storage.write_array("arr", np.arange(4))
+        self.storage.write_attrs("arr", {"key": "value"})  # write extra information
         return self.parameters["a"] * self.parameters["b"]
 
 
 # create an instance of the model
-model = MyModel({"a": 3})
+model = MyModel({"a": 3}, output="test.yaml")
 # run the instance
 print(model())

py-modelrunner-0.14.0/examples/model_storage_output.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+This example shows defining a custom model that stores additional data
+
+.. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
+"""
+
+import tempfile
+
+from modelrunner import make_model, open_storage
+
+
+def multiply(a, b=2, storage=None):
+    storage["data"] = {"additional": "information"}
+    return a * b
+
+
+with tempfile.NamedTemporaryFile(suffix=".yaml") as fp:
+    # create an instance of the model defined by the function
+    model = make_model(multiply, {"a": 3}, output=fp.name)
+    # run the instance and store the data
+    model.write_result()
+
+    # read the file and check whether all the data is there
+    with open_storage(fp.name) as storage:
+        print("Stored data:", storage["data"])
+        print("Model result:", storage["result"].data)

{py-modelrunner-0.13.0 → py-modelrunner-0.14.0}/examples/script_function.py

@@ -15,4 +15,4 @@ def multiply(a: float = 1, b: float = 2):
 
 if __name__ == "__main__":
     result = run_function_with_cmd_args(multiply)
-    print(result.state)
+    print(result.data)

{py-modelrunner-0.13.0 → py-modelrunner-0.14.0}/modelrunner/__init__.py

@@ -1,15 +1,4 @@
 """
-
-.. inheritance-diagram::
-        state.io.IOBase
-        state.base.StateBase
-        state.array.ArrayState
-        state.array_collection.ArrayCollectionState
-        state.object.ObjectState
-        state.dict.DictState
-        results.Result
-   :parts: 1
-
 .. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
 """
 
@@ -41,11 +30,4 @@ from .model import (
 from .parameters import HideParameter, Parameter, Parameterized
 from .results import Result, ResultCollection
 from .run import submit_job, submit_jobs
-from .state import (
-    ArrayCollectionState,
-    ArrayState,
-    DictState,
-    ObjectState,
-    load_state,
-    make_state,
-)
+from .storage import open_storage

py-modelrunner-0.14.0/modelrunner/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.14.0'
+__version_tuple__ = version_tuple = (0, 14, 0)

py-modelrunner-0.14.0/modelrunner/compatibility/__init__.py

@@ -0,0 +1,5 @@
+"""
+Code for maintaining backwards compatibility, particularly for loading results
+
+.. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
+"""

py-modelrunner-0.14.0/modelrunner/compatibility/triage.py

@@ -0,0 +1,181 @@
+"""
+Contains code necessary for deciding which format version was used to write a file
+
+.. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Mapping, Optional, Union
+
+from ..model import ModelBase
+from ..results import Result
+
+if TYPE_CHECKING:
+    from zarr.storage import BaseStore
+
+
+Store = Union[str, Path, "BaseStore"]
+
+
+def guess_format(path: Path) -> str:
+    """guess the format of a given store
+
+    Args:
+        path (str or :class:`~pathlib.Path`):
+            Path pointing to a file
+
+    Returns:
+        str: The store format
+    """
+    # guess format from path extension
+    ext = Path(path).suffix.lower()
+    if ext == ".json":
+        return "json"
+    elif ext in {".yml", ".yaml"}:
+        return "yaml"
+    elif ext in {".h5", ".hdf", ".hdf5"}:
+        return "hdf"
+    else:
+        return "zarr"  # fallback to the default storage method based on zarr
+
+
+def normalize_zarr_store(store: Store, mode: str = "a") -> Optional[Store]:
+    """determine best file format for zarr storage
+
+    In particular, we use a :class:`~zarr.storage.ZipStore` when a path looking like a
+    file is given.
+
+    Args:
+        store: User-provided store
+        mode (str): The mode with which the file will be opened
+
+    Returns:
+    """
+    import zipfile
+
+    import zarr
+
+    if isinstance(store, (str, Path)):
+        store = Path(store)
+        if store.is_file():
+            try:
+                store = zarr.storage.ZipStore(store, mode=mode)
+            except zipfile.BadZipfile:
+                return None
+        else:
+            return None
+    return store
+
+
+def _find_version(data: Mapping[str, Any], label: str) -> Optional[int]:
+    """try finding version information in different places in `data`
+
+    Args:
+        data (dict):
+            A mapping that contains attribute information
+        label (str):
+            The label of the data that should be loaded
+
+    Returns:
+        int: The format version or None if it could not be found
+    """
+
+    def read_version(item) -> Optional[str]:
+        """try reading attribute from a particular item"""
+        if hasattr(item, "attrs"):
+            return read_version(item.attrs)
+        elif "__version__" in item:
+            return item["__version__"]  # type: ignore
+        elif "format_version" in item:
+            return item["format_version"]  # type: ignore
+        elif "__attrs__" in item:
+            return read_version(item["__attrs__"])
+        elif "attributes" in item:
+            return read_version(item["attributes"])
+        else:
+            return None
+
+    format_version = read_version(data)
+    if format_version is None and label in data:
+        format_version = read_version(data[label])
+    if format_version is None and "state" in data:
+        format_version = read_version(data["state"])
+
+    if isinstance(format_version, str):
+        return json.loads(format_version)  # type: ignore
+    else:
+        return format_version
+
+
+def result_check_load_old_version(
+    path: Path, loc: str, *, model: Optional[ModelBase] = None
+) -> Optional[Result]:
+    """check whether the resource can be loaded with an older version of the package
+
+    Args:
+        path (str or :class:`~pathlib.Path`):
+            The path to the resource to be loaded
+        loc (str):
+            Label, key, or location of the item to be loaded
+        model (:class:`~modelrunner.model.ModelBase`, optional):
+            Optional model that was used to write this result
+
+    Returns:
+        :class:`~modelrunner.result.Result`:
+            The loaded result or `None` if we cannot load it with the old versions
+    """
+    label = "data" if loc is None else loc
+    format_version = None
+    # check for compatibility
+    fmt = guess_format(path)
+    if fmt == "json":
+        with open(path, mode="r") as fp:
+            format_version = _find_version(json.load(fp), label)
+
+    elif fmt == "yaml":
+        import yaml
+
+        with open(path, mode="r") as fp:
+            format_version = _find_version(yaml.safe_load(fp), label)
+
+    elif fmt == "hdf":
+        import h5py
+
+        with h5py.File(path, mode="r") as root:
+            format_version = _find_version(root, label)
+
+    elif fmt == "zarr":
+        import zarr
+
+        store = normalize_zarr_store(path, mode="r")
+        if store is None:
+            return None  # could not open zarr file
+        with zarr.open_group(store, mode="r") as root:
+            format_version = _find_version(root, label)
+            if format_version is None and label != "data":
+                format_version = _find_version(root, "data")
+                if format_version is not None:
+                    label = "data"
+
+    else:
+        return None
+
+    if format_version in {0, None}:
+        # load result written with format version 0
+        from .version0 import result_from_file_v0
+
+        return result_from_file_v0(path, model=model)
+
+    elif format_version == 1:
+        # load result written with format version 1
+        from .version1 import result_from_file_v1
+
+        return result_from_file_v1(path, label=label, model=model)
+
+    elif not isinstance(format_version, int):
+        raise RuntimeError(f"Unsupported format version {format_version}")
+
+    return None

py-modelrunner-0.14.0/modelrunner/compatibility/version0.py

@@ -0,0 +1,101 @@
+"""
+Contains code necessary for loading results from format version 0
+
+.. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Mapping, Optional
+
+import numpy as np
+
+from ..model import ModelBase
+from ..results import Result
+from .triage import guess_format
+
+
+def read_hdf_data(node):
+    """read structured data written with :func:`write_hdf_dataset` from an HDF node"""
+    import h5py
+
+    if isinstance(node, h5py.Dataset):
+        return np.array(node)
+    else:
+        # this must be a group
+        data = {key: json.loads(value) for key, value in node.attrs.items()}
+        for key, value in node.items():
+            data[key] = read_hdf_data(value)
+        return data
+
+
+def _Result_from_simple_objects(
+    content: Mapping[str, Any], model: Optional[ModelBase] = None
+) -> Result:
+    """read result from simple object (like loaded from a JSON file) using version 0
+
+    Args:
+        content (dict):
+            Data from which the result is restored
+        model (:class:`ModelBase`):
+            Model associated with the result
+
+    Returns:
+        :class:`Result`: the restored result
+    """
+
+    return Result.from_data(
+        model_data=content.get("model", {}),
+        result=content.get("result"),
+        model=model,
+        info=content.get("info", {}),
+    )
+
+
+def _Result_from_hdf(hdf_element, model: Optional[ModelBase] = None) -> Result:
+    """old reader for backward compatible reading"""
+    model_data = {key: json.loads(value) for key, value in hdf_element.attrs.items()}
+    if "result" in hdf_element:
+        result = read_hdf_data(hdf_element["result"])
+    else:
+        result = model_data.pop("result")
+    # check for other nodes, which might not be read
+
+    info = model_data.pop("__info__") if "__info__" in model_data else {}
+
+    return Result.from_data(
+        model_data=model_data, result=result, model=model, info=info
+    )
+
+
+def result_from_file_v0(path: Path, **kwargs) -> Result:
+    """load object from a file using format version 1
+
+    Args:
+        store (str or :class:`zarr.Store`):
+            Path or instance describing the storage, which is either a file path or
+            a :class:`zarr.Storage`.
+    """
+    fmt = guess_format(path)
+    if fmt == "json":
+        with open(path, mode="r") as fp:
+            content = json.load(fp)
+        return _Result_from_simple_objects(content, **kwargs)
+
+    elif fmt == "yaml":
+        import yaml
+
+        with open(path, mode="r") as fp:
+            content = yaml.safe_load(fp)
+        return _Result_from_simple_objects(content, **kwargs)
+
+    elif fmt == "hdf":
+        import h5py
+
+        with h5py.File(path, mode="r") as root:
+            return _Result_from_hdf(root, **kwargs)
+
+    else:
+        raise NotImplementedError(f"Format `{fmt}` not implemented")