ezarr 1.0.0__tar.gz

ezarr-1.0.0/.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+*.vim

ezarr-1.0.0/.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+fail_fast: true
+
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.11.13
+  hooks:
+    # Run the linter.
+    - id: ruff-check
+      name: ruff fix
+      args: [ --fix, --output-format, concise ]
+    # Fix old typing and syntax
+    - id: ruff-check
+      name: ruff update
+      args: [ --fix, --select, UP,F401, --output-format, concise ]
+    # Run the formatter.
+    - id: ruff-format

ezarr-1.0.0/.python-version

@@ -0,0 +1 @@
+3.12

ezarr-1.0.0/PKG-INFO

@@ -0,0 +1,132 @@
+Metadata-Version: 2.4
+Name: ezarr
+Version: 1.0.0
+Summary: Tools for easily working with Zarr groups and arrays
+Author-email: Matteo Bouvier <matteo.bouvier@lyon.unicancer.fr>
+Requires-Python: >=3.12
+Requires-Dist: zarr>=3
+Description-Content-Type: text/markdown
+
+# eZarr
+
+Easily store any Python object inside [zarr](https://zarr.readthedocs.io/en/latest/) hierarchies.
+This package defines proxies for regular Python lists (EZList) and dicts (EZDict), built as wrappers around zarr Arrays and Groups.
+It also defines a Protocol for writing and loading any object in zarr hierarchies.
+
+## EZList
+An EZlist is a proxy for a Python list built on top of a zarr.Array.
+It allows storing mixed arbitrary objects in a 1 dimensional sequence.
+
+### Examples
+```python
+>>> from ezarr import EZList
+>>> from pathlib import Path
+
+# EZList creation from a zarr.Array:
+>>> EZList(zarr.array([1, 2, 3]))
+EZList[1, 2, 3]
+
+# EZList creation from a list:
+>>> data = [1, "some text", Path("some/path.txt")]
+>>> ez_list = EZList.from_list(data, store={}, name="data")
+                                      #    ~~        ~~~~ a name for the list inside the store
+                                      #    |
+                                      #    or any zarr StoreLike object to store the list
+
+>>> ez_list
+EZList[1, 'some text', PosixPath('some/path.txt')]
+
+# EZLists support general list methods:
+>>> ez_list[0]
+1
+
+>>> del ez_list[1]
+>>> ez_list
+EZList[1, PosixPath('some/path.txt')]
+
+>>> ez_list.append(3.14)
+>>> ez_list
+EZList[1, PosixPath('some/path.txt'), 3.14]
+
+>>> ez_list.insert(0, [1, 2, 3])
+>>> ez_list
+EZList[[1, 2, 3], 1, PosixPath('/some/path.txt'), 3.14]
+
+# EZList save to zarr store:
+>>> ez_list.save("path/to/store", name="saved_list")
+
+# EZList load from zarr store:
+>>> EZList.open("path/to/store", name="saved_list")
+EZList[[1, 2, 3], 1, PosixPath('/some/path.txt'), 3.14]
+
+# EZList conversion to a regular list:
+>>> ez_list.copy()
+[[1, 2, 3], 1, PosixPath('some/path.txt'), 3.14]
+```
+
+## EZDict
+An EZDict is a proxy for a Python dict built on top of a zarr.Group.
+It expands zarr.Groups to store and retrieve arbitrary objects.
+
+### Examples
+```python
+>>> from ezarr import EZDict
+
+# EZdDct creation for a zarr.Group:
+>>> EZDict(zarr.open_group({}))
+EZDict{}
+
+# EZDict creation for a dict:
+>>> data = {"a": 1, "b": [1, 2, 3], "c": {"d": "some text"}}
+>>> ez_dict = EZDict.from_dict(data, store={})
+                                    #      ~~ or any zarr StoreLike object to store the dict into                                    #      |
+
+# EZDict lazy representation:
+>>> ez_dict
+EZDict{
+        a: 1,
+        b: [1 2 3],
+        c: {...}                    # <-- note the nested dict was not loaded for the representation
+}
+
+# EZDict save to zarr store:
+>>> ez_dict.save("path/to/store", name="saved_dict")
+
+# EZDict load from zarr store:
+>>> EZDict.open("path/to/store", name="saved_dict")
+EZDict{
+        a: 1,
+        b: [1 2 3],
+        c: {...}
+}
+
+# EZDict conversion to a regular dict:
+>>> ez_dict.copy()
+{'a': np.int64(1), 'b': array([1, 2, 3]), 'c': {'d': np.str_('some text')}}
+
+```
+
+## SupportsEZReadWrite
+The `SupportsEZReadWrite` protocol, once implemented on an object, allows it to be store in and retrieved from a zarr hierarchy.
+
+```python
+@runtime_checkable
+class SupportsEZRead(Protocol):
+    @classmethod
+    def __ez_read__(cls, grp: zarr.Group) -> Self: ...
+
+
+@runtime_checkable
+class SupportsEZWrite(Protocol):
+    def __ez_write__(self, grp: zarr.Group) -> None: ...
+
+
+@runtime_checkable
+class SupportsEZReadWrite(SupportsEZRead, SupportsEZWrite, Protocol): ...
+```
+
+This protocol has 2 parts:
+ - `SupportsEZRead` defines the classmethod `__ez_read__(cls, grp: zarr.Group) -> Self` for reconstructing an object from a zarr.Group.
+ - `SupportsEZWrite` defines the method `__ez_write__(self, grp: zarr.Group) -> None` for storing an object in a zarr.Group.
+
+ Once implemented on an object, this protocol allows that object to be automatically saved and loaded in EZLists and EZDicts.

ezarr-1.0.0/README.md

@@ -0,0 +1,123 @@
+# eZarr
+
+Easily store any Python object inside [zarr](https://zarr.readthedocs.io/en/latest/) hierarchies.
+This package defines proxies for regular Python lists (EZList) and dicts (EZDict), built as wrappers around zarr Arrays and Groups.
+It also defines a Protocol for writing and loading any object in zarr hierarchies.
+
+## EZList
+An EZlist is a proxy for a Python list built on top of a zarr.Array.
+It allows storing mixed arbitrary objects in a 1 dimensional sequence.
+
+### Examples
+```python
+>>> from ezarr import EZList
+>>> from pathlib import Path
+
+# EZList creation from a zarr.Array:
+>>> EZList(zarr.array([1, 2, 3]))
+EZList[1, 2, 3]
+
+# EZList creation from a list:
+>>> data = [1, "some text", Path("some/path.txt")]
+>>> ez_list = EZList.from_list(data, store={}, name="data")
+                                      #    ~~        ~~~~ a name for the list inside the store
+                                      #    |
+                                      #    or any zarr StoreLike object to store the list
+
+>>> ez_list
+EZList[1, 'some text', PosixPath('some/path.txt')]
+
+# EZLists support general list methods:
+>>> ez_list[0]
+1
+
+>>> del ez_list[1]
+>>> ez_list
+EZList[1, PosixPath('some/path.txt')]
+
+>>> ez_list.append(3.14)
+>>> ez_list
+EZList[1, PosixPath('some/path.txt'), 3.14]
+
+>>> ez_list.insert(0, [1, 2, 3])
+>>> ez_list
+EZList[[1, 2, 3], 1, PosixPath('/some/path.txt'), 3.14]
+
+# EZList save to zarr store:
+>>> ez_list.save("path/to/store", name="saved_list")
+
+# EZList load from zarr store:
+>>> EZList.open("path/to/store", name="saved_list")
+EZList[[1, 2, 3], 1, PosixPath('/some/path.txt'), 3.14]
+
+# EZList conversion to a regular list:
+>>> ez_list.copy()
+[[1, 2, 3], 1, PosixPath('some/path.txt'), 3.14]
+```
+
+## EZDict
+An EZDict is a proxy for a Python dict built on top of a zarr.Group.
+It expands zarr.Groups to store and retrieve arbitrary objects.
+
+### Examples
+```python
+>>> from ezarr import EZDict
+
+# EZdDct creation for a zarr.Group:
+>>> EZDict(zarr.open_group({}))
+EZDict{}
+
+# EZDict creation for a dict:
+>>> data = {"a": 1, "b": [1, 2, 3], "c": {"d": "some text"}}
+>>> ez_dict = EZDict.from_dict(data, store={})
+                                    #      ~~ or any zarr StoreLike object to store the dict into                                    #      |
+
+# EZDict lazy representation:
+>>> ez_dict
+EZDict{
+        a: 1,
+        b: [1 2 3],
+        c: {...}                    # <-- note the nested dict was not loaded for the representation
+}
+
+# EZDict save to zarr store:
+>>> ez_dict.save("path/to/store", name="saved_dict")
+
+# EZDict load from zarr store:
+>>> EZDict.open("path/to/store", name="saved_dict")
+EZDict{
+        a: 1,
+        b: [1 2 3],
+        c: {...}
+}
+
+# EZDict conversion to a regular dict:
+>>> ez_dict.copy()
+{'a': np.int64(1), 'b': array([1, 2, 3]), 'c': {'d': np.str_('some text')}}
+
+```
+
+## SupportsEZReadWrite
+The `SupportsEZReadWrite` protocol, once implemented on an object, allows it to be store in and retrieved from a zarr hierarchy.
+
+```python
+@runtime_checkable
+class SupportsEZRead(Protocol):
+    @classmethod
+    def __ez_read__(cls, grp: zarr.Group) -> Self: ...
+
+
+@runtime_checkable
+class SupportsEZWrite(Protocol):
+    def __ez_write__(self, grp: zarr.Group) -> None: ...
+
+
+@runtime_checkable
+class SupportsEZReadWrite(SupportsEZRead, SupportsEZWrite, Protocol): ...
+```
+
+This protocol has 2 parts:
+ - `SupportsEZRead` defines the classmethod `__ez_read__(cls, grp: zarr.Group) -> Self` for reconstructing an object from a zarr.Group.
+ - `SupportsEZWrite` defines the method `__ez_write__(self, grp: zarr.Group) -> None` for storing an object in a zarr.Group.
+
+ Once implemented on an object, this protocol allows that object to be automatically saved and loaded in EZLists and EZDicts.

ezarr-1.0.0/ezarr/__init__.py

@@ -0,0 +1,5 @@
+from ezarr.dict import EZDict
+from ezarr.list import EZList
+from ezarr.types import PyObject, PyObjectCodec
+
+__all__ = ["EZList", "EZDict", "PyObject", "PyObjectCodec"]

ezarr-1.0.0/ezarr/_repr.py

@@ -0,0 +1,84 @@
+import itertools as it
+import math
+from typing import Any, cast
+
+import numpy as np
+import numpy.typing as npt
+import zarr  # pyright: ignore[reportMissingTypeStubs]
+
+import ezarr
+
+
+def repr_element(elem: Any, prefix: str = "") -> str:
+    if isinstance(elem, np.str_):
+        return f"'{elem}'"
+
+    if isinstance(elem, np.generic):
+        return str(elem)
+
+    if isinstance(elem, zarr.Group | ezarr.EZDict):
+        return "{...}" if len(elem) else "{}"  # pyright: ignore[reportUnknownArgumentType]
+
+    if isinstance(elem, zarr.Array):
+        return repr_array(elem, prefix=prefix)
+
+    return repr(elem)
+
+
+def repr_array(arr: zarr.Array, prefix: str = "") -> str:
+    if arr.size == 0:
+        return "[]"
+
+    if arr.ndim == 0:
+        return repr_element(arr[()])
+
+    data = cast(npt.NDArray[Any], arr.get_orthogonal_selection(tuple(_get_index(shape) for shape in arr.shape)))
+
+    if np.issubdtype(data.dtype, np.integer):
+        align = math.ceil(math.log10(data.max()))
+
+    elif np.issubdtype(data.dtype, np.floating):
+        data = np.vectorize(np.format_float_positional)(data)
+        align = np.vectorize(len)(data).max()
+
+    else:
+        data = data.astype(str)
+        align = np.vectorize(len)(data)
+
+    return _print3(data, list(arr.shape), align=align, prefix=prefix, ignore_prefix=True)
+
+
+def _get_index(axis_shape: int) -> list[int]:
+    if axis_shape <= 6:
+        return list(range(axis_shape))
+
+    return [0, 1, 2, -3, -2, -1]
+
+
+def _print3(data: np.ndarray, shape: list[int], align: int, *, prefix: str = "", ignore_prefix: bool = False) -> str:
+    if data.ndim == 1:
+        if shape[0] <= 6:
+            values = data
+        else:
+            values = it.chain(data[:3], ["..."], data[3:])
+
+        return f"{'' if ignore_prefix else prefix}[{' '.join(map(lambda v: f'{v:>{align}}', values))}]"
+
+    if shape[0] <= 6:
+        rows = [
+            _print3(data[idx], shape[1:], align, prefix=prefix + " ", ignore_prefix=idx == 0) for idx in range(shape[0])
+        ]
+
+    else:
+        rows = [
+            _print3(data[0], shape[1:], align, prefix=prefix + " ", ignore_prefix=True),
+            _print3(data[1], shape[1:], align, prefix=prefix + " "),
+            _print3(data[2], shape[1:], align, prefix=prefix + " "),
+            f"{prefix} ...",
+            _print3(data[3], shape[1:], align, prefix=prefix + " "),
+            _print3(data[4], shape[1:], align, prefix=prefix + " "),
+            _print3(data[5], shape[1:], align, prefix=prefix + " "),
+        ]
+
+    sep = "," + "\n" * max(1, len(shape) - 1)
+    return f"{'' if ignore_prefix else prefix}[{sep.join(rows)}]"

ezarr-1.0.0/ezarr/dict.py

@@ -0,0 +1,264 @@
+from __future__ import annotations
+
+import itertools as it
+from collections.abc import ItemsView, Iterator, Mapping, MutableMapping
+from typing import Any, Self, cast, override
+import warnings
+
+import numpy as np
+import numpy.typing as npt
+import zarr
+from numpy._typing import _SupportsArray as SupportsArray  # pyright: ignore[reportPrivateUsage]
+from zarr.errors import UnstableSpecificationWarning
+from zarr.storage import StoreLike  # pyright: ignore[reportUnknownVariableType]
+
+import ezarr
+from ezarr import io
+from ezarr._repr import repr_element
+from ezarr.names import AccessModeLiteral
+from ezarr.object import EZObject
+
+type DictData[T] = Mapping[str, int | float | np.integer | np.floating | list[Any] | npt.ArrayLike | DictData[T]]
+type GroupItems = ItemsView[str, zarr.Group | zarr.Array]
+
+
+class EZDict[T](EZObject[T, dict[str, T]], MutableMapping[str, T]):
+    """
+    Dict-like object wrapping a zarr.Group for storing arbitrary Python objects
+
+    Args:
+        group: a zarr.Group
+
+    Example:
+        >>> EZDict(zarr.open_group({}))
+        EZDict{}
+    """
+
+    def __init__(self, group: zarr.Group) -> None:
+        self._group: zarr.Group = group
+
+    @classmethod
+    def from_dict(
+        cls,
+        dct: Mapping[str, Any],
+        *,
+        name: str = "",
+        store: StoreLike | None = None,
+        mode: AccessModeLiteral = "a",
+        path: str | None = None,
+        overwrite: bool = False,
+    ) -> Self:
+        r"""
+        Create an EZDict from a regular in-memory Python dictionary.
+
+        Args:
+            dct: dictionary with arbitrary data to store.
+            name: name for the EZDict, to use inside the store.
+            store: Store or path to directory in file system or nam of zip file.
+            mode: Persistence mode, in ['r', 'r+', 'a', 'w', 'w-'].
+            path: Group path within store.
+            overwrite: overwrite object if a group with name `name` already exists ? (default: False)
+
+        Example:
+            >>> data = {"a": 1, "b": [1, 2, 3], "c": {"d": "some text"}}
+            >>> ez_dict = EZDict.from_dict(data)
+            >>> repr(ez_dict)
+            'EZDict{\n\ta: 1,\n\tb: [1 2 3],\n\tc: {...}\n}'
+        """
+        if store is None:
+            store = {}
+
+        grp = zarr.open_group(store, mode=mode, path=path)
+
+        if name:
+            grp = grp.create_group(name, overwrite=overwrite)
+
+        to_visit: list[tuple[Any, str, str]] = [(dct[k], k, "") for k in dct.keys()]
+
+        while len(to_visit):
+            value, name_, path_ = to_visit.pop()
+
+            if isinstance(value, Mapping):
+                to_visit.extend([(value[k], k, f"{path_}/{name_}" if path_ else name_) for k in value.keys()])  # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType]
+
+            elif isinstance(value, list | SupportsArray):
+                grp.require_group(path_).create_array(name_, data=np.asarray(value), overwrite=overwrite)  # pyright: ignore[reportUnknownArgumentType]
+
+            else:
+                io.write_object(grp, obj=value, name=name_, path=path_, overwrite=overwrite)
+
+        return cls(grp)
+
+    @staticmethod
+    def _repr(grp: zarr.Group) -> str:
+        if not len(grp):
+            return "{}"
+
+        if len(grp) > 100:
+            return (
+                "{\n\t"
+                + ",\n\t".join(
+                    [
+                        f"{name}: {repr_element(io.read_object(grp, name=name), prefix=f'\t{" " * len(name)}  ')}"
+                        for name in it.islice(sorted(grp.keys()), 0, 10)
+                    ]
+                )
+                + ",\n\t...,\n\t"
+                + ",\n\t".join(
+                    [
+                        f"{name}: {repr_element(io.read_object(grp, name=name), prefix=f'\t{" " * len(name)}  ')}"
+                        for name in it.islice(sorted(grp.keys()), len(grp) - 10, None)
+                    ]
+                )
+                + "}"
+            )
+
+        return (
+            "{\n\t"
+            + ",\n\t".join(
+                [
+                    f"{name}: {repr_element(io.read_object(grp, name=name), prefix=f'\t{" " * len(name)}  ')}"
+                    for name in sorted(grp.keys())
+                ]
+            )
+            + "\n}"
+        )
+
+    @override
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}{self._repr(self._group)}"
+
+    @override
+    def __len__(self) -> int:
+        return len(self._group)
+
+    @override
+    def __getitem__(self, key: str, /) -> T:
+        return io.read_object(self._group, name=key)
+
+    @override
+    def __setitem__(self, key: str, value: T, /) -> None:
+        if key in self:
+            try:
+                # try to compare values, might fail when comparing arrays with different shapes
+                if self[key] == value:
+                    return
+
+            except ValueError:
+                pass
+
+        with warnings.catch_warnings(action="ignore", category=UnstableSpecificationWarning):
+            io.write_object(self._group, obj=value, name=key, overwrite=True)
+
+    @override
+    def __delitem__(self, key: str, /) -> None:
+        del self._group[key]
+
+    @override
+    def __iter__(self) -> Iterator[str]:
+        yield from sorted(self._group.keys())
+
+    def __deepcopy__(self, _memo: dict[Any, Any]) -> dict[str, Any]:
+        return self.copy()
+
+    def __ior__(self, other: object) -> EZDict[T]:
+        if not isinstance(other, Mapping):
+            raise NotImplementedError
+
+        other = cast(Mapping[str, T], other)
+        for name, value in other.items():
+            if isinstance(value, Mapping):
+                grp = self._group.require_group(name)
+                EZDict(grp).__ior__(value)  # pyright: ignore[reportUnknownArgumentType]
+
+            else:
+                with warnings.catch_warnings(action="ignore", category=UnstableSpecificationWarning):
+                    self[name] = value
+
+        return self
+
+    @classmethod
+    @override
+    def open(
+        cls, store: StoreLike | None = None, *, name: str = "", mode: AccessModeLiteral = "a", path: str | None = None
+    ) -> Self:
+        r"""
+        Open this EZDict from a store.
+
+        Args:
+            store: Store, path to a directory or name of a zip file.
+            name: name for the EZDict, to use inside the store.
+            mode: Persistence mode.
+            path: path within the store to open.
+
+        Example:
+            >>> ez_dict = EZDict.from_dict({"a": 1, "b": [1, 2, 3], "c": {"d": "some text"}})
+            >>> ez_dict.save("/tmp/dict", overwrite=True)
+            >>> repr(ez_dict.open("/tmp/dict"))
+            'EZDict{\n\ta: 1,\n\tb: [1 2 3]\n}'
+        """
+        path = f"{path.rstrip('/')}/{name}" if path else name
+        return cls(zarr.open_group(store, mode=mode, path=path))
+
+    @override
+    def save(
+        self,
+        store: StoreLike,
+        *,
+        name: str = "",
+        mode: AccessModeLiteral = "a",
+        path: str | None = None,
+        overwrite: bool = False,
+    ) -> None:
+        """
+        Save this EZDict to a local file system.
+
+        Args:
+            store: Store, path to a directory or name of a zip file.
+            name: name for the EZDict, to use inside the store.
+            mode: Persistence mode.
+            path: path within the store where the EZDict will be saved.
+            overwrite: overwrite EZDict if a group with name `name` already exists ? (default: False)
+
+        Example:
+            >>> from pathlib import Path
+            >>> ez_dict = EZDict.from_dict({"a": 1, "b": [1, 2, 3], "c": {"d": "some text"}})
+            >>> ez_dict.save("/tmp/dict", overwrite=True)
+            >>> Path("/tmp/dict").exists()
+            True
+        """
+        path = f"{path.rstrip('/')}/{name}/" if path else f"{name}/"
+
+        for _, array in self._group.arrays():
+            assert isinstance(array, zarr.Array)
+            zarr.create_array(store, name=path + array.path, data=array, overwrite=overwrite)  # pyright: ignore[reportArgumentType]
+
+    @staticmethod
+    def _copy_nested(dct: dict[str, Any], value: EZDict[Any]) -> dict[str, Any]:
+        for k, v in value.items():
+            if isinstance(v, EZDict):
+                sub = dct.setdefault(k, {})
+                EZDict._copy_nested(sub, v)  # pyright: ignore[reportUnknownArgumentType]
+
+            elif isinstance(v, ezarr.EZList):
+                dct[k] = v.copy()
+
+            elif isinstance(v, zarr.Array):
+                dct[k] = np.array(v)
+
+            else:
+                dct[k] = v
+
+        return dct
+
+    @override
+    def copy(self) -> dict[str, T]:
+        """
+        Convert this EZDict into a Python dict, loading all the data into memory.
+
+        Example:
+            >>> ez_dict = EZDict.from_dict({"a": 1, "b": [1, 2, 3], "c": {"d": "some text"}})
+            >>> ez_dict.copy()
+            {'a': np.int64(1), 'b': array([1, 2, 3]), 'c': {'d': np.str_('some text')}}
+        """
+        return EZDict._copy_nested({}, self)