zarr-n5 0.1.0__tar.gz

zarr_n5-0.1.0/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.3
+Name: zarr-n5
+Version: 0.1.0
+Summary: Utilities for accessing N5 data through zarr v3.
+Author: Chris Barnes
+Author-email: Chris Barnes <chris.barnes@gerbi-gmb.de>
+Requires-Dist: zarr>=3.1.5
+Requires-Python: >=3.12, <4.0
+Description-Content-Type: text/markdown
+
+# zarr-python-n5
+
+[N5](https://github.com/saalfeldlab/n5) utilities for [zarr-python](https://github.com/zarr-developers/zarr-python).
+
+- Documentation: <https://zarr-python-n5.readthedocs.io>
+
+## Codecs
+
+### N5 Default Codec
+
+[As described here](https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/n5_default).
+
+Only whole-chunk reading is supported.
+
+#### N5 Compressor support
+
+| N5 compressor | Supported | Zarr bytes-to-bytes codec | Notes |
+| ------------- | --------- | ------------------------- | ----- |
+| `raw` | yes | n/a | Equivalent to omitted bytes-to-bytes codec |
+| `blosc` | yes | `blosc` | |
+| `gzip` | yes | `gzip` | |
+| `zstd` | yes | `zstd` | |
+| `lz4` | no | | [Incompatible codecs](https://github.com/zarr-developers/numcodecs/issues/175) |
+| `xz` | no | | No equivalent Zarr codec |
+| `jpeg` | no | | Needs [N5 documentation](https://github.com/saalfeldlab/n5-jpeg/issues/1), [Zarr codec](https://github.com/zarr-developers/zarr-extensions/issues/15) |
+| `bzip2` | no | | No equivalent Zarr codec |
+
+## Stores
+
+`N5WrapperStore` allows reading N5 data with DEFAULT-mode blocks through any Zarr store by converting metadata on the fly.
+By default, this does not replicate the N5 behaviour of inferring an empty group where a metadata document does not exist.
+To achieve this, wrap it in the provided `ImplicitGroupWrapperStore`.
+
+## Tools
+
+This package provides `n5tozarr`, a command-line interface for converting N5 data to Zarr in-place.
+The N5 metadata are left untouched, and no chunk data is altered, moved, or copied.
+A `zarr.json` file is simply added to each Zarr node.
+
+N5 attributes are extracted and added to the `zarr.json` attributes.
+
+The full N5 metadata document is accessible inside the `zarr.json` in an attribute called `_n5`.
+If a directory/prefix was empty and the existence of an N5 group was inferred,
+the `zarr.json` attribute `_implicit` will be `true`.
+
+## Contributing
+
+Use [`uv`](https://docs.astral.sh/uv/) for project management.
+
+Use [`just`](https://github.com/casey/just) for common development tasks.

zarr_n5-0.1.0/README.md

@@ -0,0 +1,50 @@
+# zarr-python-n5
+
+[N5](https://github.com/saalfeldlab/n5) utilities for [zarr-python](https://github.com/zarr-developers/zarr-python).
+
+- Documentation: <https://zarr-python-n5.readthedocs.io>
+
+## Codecs
+
+### N5 Default Codec
+
+[As described here](https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/n5_default).
+
+Only whole-chunk reading is supported.
+
+#### N5 Compressor support
+
+| N5 compressor | Supported | Zarr bytes-to-bytes codec | Notes |
+| ------------- | --------- | ------------------------- | ----- |
+| `raw` | yes | n/a | Equivalent to omitted bytes-to-bytes codec |
+| `blosc` | yes | `blosc` | |
+| `gzip` | yes | `gzip` | |
+| `zstd` | yes | `zstd` | |
+| `lz4` | no | | [Incompatible codecs](https://github.com/zarr-developers/numcodecs/issues/175) |
+| `xz` | no | | No equivalent Zarr codec |
+| `jpeg` | no | | Needs [N5 documentation](https://github.com/saalfeldlab/n5-jpeg/issues/1), [Zarr codec](https://github.com/zarr-developers/zarr-extensions/issues/15) |
+| `bzip2` | no | | No equivalent Zarr codec |
+
+## Stores
+
+`N5WrapperStore` allows reading N5 data with DEFAULT-mode blocks through any Zarr store by converting metadata on the fly.
+By default, this does not replicate the N5 behaviour of inferring an empty group where a metadata document does not exist.
+To achieve this, wrap it in the provided `ImplicitGroupWrapperStore`.
+
+## Tools
+
+This package provides `n5tozarr`, a command-line interface for converting N5 data to Zarr in-place.
+The N5 metadata are left untouched, and no chunk data is altered, moved, or copied.
+A `zarr.json` file is simply added to each Zarr node.
+
+N5 attributes are extracted and added to the `zarr.json` attributes.
+
+The full N5 metadata document is accessible inside the `zarr.json` in an attribute called `_n5`.
+If a directory/prefix was empty and the existence of an N5 group was inferred,
+the `zarr.json` attribute `_implicit` will be `true`.
+
+## Contributing
+
+Use [`uv`](https://docs.astral.sh/uv/) for project management.
+
+Use [`just`](https://github.com/casey/just) for common development tasks.

zarr_n5-0.1.0/pyproject.toml

@@ -0,0 +1,37 @@
+[project]
+name = "zarr-n5"
+version = "0.1.0"
+description = "Utilities for accessing N5 data through zarr v3."
+readme = "README.md"
+authors = [{ name = "Chris Barnes", email = "chris.barnes@gerbi-gmb.de" }]
+requires-python = ">=3.12,<4.0"
+dependencies = ["zarr>=3.1.5"]
+
+[project.entry-points."zarr.codecs"]
+"n5_default" = "zarr_n5:N5DefaultCodec"
+
+[project.scripts]
+n5tozarr = "zarr_n5.cli.convert:main"
+
+
+[build-system]
+requires = ["uv_build>=0.9.8,<0.10.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    { include-group = "lint" },
+    { include-group = "test" },
+    { include-group = "doc" },
+]
+doc = [
+    "pdoc>=16.0.0",
+]
+lint = [
+    "mypy>=1.19.1",
+    "ruff>=0.15.6",
+]
+test = [
+    "pytest>=9.0.2",
+    "tensorstore>=0.1.82",
+]

zarr_n5-0.1.0/src/zarr_n5/__init__.py

@@ -0,0 +1,13 @@
+"""
+Utilities for working with [N5](https://github.com/saalfeldlab/n5) data through [zarr-python](https://github.com/zarr-developers/zarr-python).
+"""
+
+from zarr.registry import register_codec
+
+from .codec.default import N5DefaultCodec
+from .storage.n5 import N5WrapperStore
+from .storage.implicit import ImplicitGroupWrapperStore
+
+__all__ = ["N5WrapperStore", "ImplicitGroupWrapperStore", "N5DefaultCodec"]
+
+register_codec("n5_default", N5DefaultCodec, qualname="zarr_n5.N5DefaultCodec")

zarr_n5-0.1.0/src/zarr_n5/cli/__init__.py

@@ -0,0 +1 @@
+"""Modules for command line interfaces."""

zarr_n5-0.1.0/src/zarr_n5/cli/convert.py

@@ -0,0 +1,33 @@
+"""n5tozarr command line interface."""
+
+from argparse import ArgumentParser
+import asyncio
+
+from ..convert import convert_hierarchy, DEFAULT_TASKS
+
+
+def main(raw_args=None):
+    """n5tozarr main function."""
+    parser = ArgumentParser("n5tozarr")
+    parser.add_argument("url", help="URL to Zarr store, using fsspec format")
+    parser.add_argument(
+        "path", help="paths within the Zarr store to process", nargs="?"
+    )
+    parser.add_argument(
+        "-t", "--tasks", type=int, default=DEFAULT_TASKS, help="asynchronous task count"
+    )
+    parser.add_argument(
+        "-d", "--max-depth", type=int, help="how far to recurse; default no maximum"
+    )
+    parser.add_argument(
+        "-I",
+        "--no-infer-groups",
+        action="store_true",
+        help="do not infer N5 groups from empty directories/ prefixes",
+    )
+
+    args = parser.parse_args(raw_args)
+    fut = convert_hierarchy(
+        args.url, args.path or "", not args.no_infer_groups, args.max_depth, args.tasks
+    )
+    asyncio.run(fut)

zarr_n5-0.1.0/src/zarr_n5/codec/__init__.py

@@ -0,0 +1,5 @@
+"""Zarr codecs."""
+
+from .default import N5DefaultCodec
+
+__all__ = ["N5DefaultCodec"]

zarr_n5-0.1.0/src/zarr_n5/codec/default.py

@@ -0,0 +1,262 @@
+"""
+N5 Default codec module.
+"""
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Self
+
+from zarr.abc.codec import ArrayBytesCodec, Codec, BytesBytesCodec, CodecPipeline
+from zarr.core.array_spec import ArraySpec
+from zarr.core.buffer.core import Buffer, NDBuffer
+from zarr.core.chunk_grids import ChunkGrid
+from zarr.core.dtype.wrapper import TBaseDType, ZDType, TBaseScalar
+from zarr.core.common import JSON, parse_named_configuration
+from zarr.core.metadata.v3 import parse_codecs
+from zarr.codecs import BytesCodec, Endian, TransposeCodec
+from zarr.registry import get_pipeline_class
+
+from ..metadata import COMPATIBLE_DATA_TYPES
+
+from ..util import N5BlockHeader
+
+__all__ = ["N5DefaultCodec"]
+
+N5_DEFAULT_NAME = "n5_default"
+N5_ENDIAN = Endian.big
+
+
+def check_valid_transpose(codec: Codec):
+    if not isinstance(codec, TransposeCodec):
+        raise ValueError("not transpose codec")
+    if codec.order != tuple(sorted(codec.order, reverse=True)):
+        raise ValueError("not a full transpose")
+
+
+def check_valid_bytes(codec: Codec):
+    if not isinstance(codec, BytesCodec):
+        raise ValueError("not bytes codec")
+    if codec.endian is not None and codec.endian != N5_ENDIAN:
+        raise ValueError("bytes codec must be big-endian")
+
+
+def check_valid_compressor(codec: Codec):
+    if not isinstance(codec, BytesBytesCodec):
+        raise ValueError("codec is not bytes-to-bytes")
+
+
+CodecTuple = (
+    tuple[TransposeCodec, BytesCodec]
+    | tuple[TransposeCodec, BytesCodec, BytesBytesCodec]
+)
+
+
+@dataclass(frozen=True)
+class N5DefaultCodec(ArrayBytesCodec):
+    """Zarr codec for default-mode N5 data.
+
+    Only full-chunk reads are supported.
+    Use the `N5DefaultCodec.from_compressor` constructor if initialising manually.
+
+    - Reads and validates the N5 block header
+    - Applies the wrapped codecs to the N5 block body
+    - Truncates or pads the resulting array to match the requested chunk
+
+    Should be the only codec present.
+    """
+
+    codecs: CodecTuple
+    """Codecs to be applied to the N5 block body."""
+
+    def __init__(self, *, codecs: Iterable[Codec | dict[str, JSON]]) -> None:
+        cs = parse_codecs(codecs)
+        if not 2 <= len(cs) <= 3:
+            raise ValueError(f"expected 2-3 codecs, got {len(cs)}")
+        check_valid_transpose(cs[0])
+        check_valid_bytes(cs[1])
+        if len(cs) > 2:
+            check_valid_compressor(cs[2])
+
+        object.__setattr__(self, "codecs", cs)
+
+    @property
+    def codec_pipeline(self) -> CodecPipeline:
+        """Get the `CodecPipeline` comprising the wrapped codecs."""
+        return get_pipeline_class().from_codecs(self.codecs)
+
+    @classmethod
+    def from_compressor(cls, ndim: int, compressor: BytesBytesCodec | None = None):
+        """Construct the codec from minimal information."""
+        transpose = cls.make_transpose(ndim)
+        endian = cls.make_bytes()
+        codecs: CodecTuple
+        if compressor is None:
+            codecs = (transpose, endian)
+        else:
+            codecs = (transpose, endian, compressor)
+        return cls(codecs=codecs)
+
+    @classmethod
+    def make_transpose(cls, ndim: int) -> TransposeCodec:
+        """Generate the `TransposeCodec` needed for this data.
+
+        N5 data is always fully transposed.
+        """
+        order = list(range(ndim))
+        return TransposeCodec(order=tuple(reversed(order)))
+
+    @classmethod
+    def make_bytes(cls) -> BytesCodec:
+        """
+        Generate the `BytesCodec` needed for this data.
+
+        N5 data is always big-endian.
+        """
+        return BytesCodec(endian=N5_ENDIAN)
+
+    def compute_encoded_size(
+        self, input_byte_length: int, chunk_spec: ArraySpec
+    ) -> int:
+        header_length = N5BlockHeader.calc_size(chunk_spec.ndim, False)
+
+        for c in self.codecs:
+            input_byte_length = c.compute_encoded_size(input_byte_length, chunk_spec)
+            chunk_spec = c.resolve_metadata(chunk_spec)
+
+        return input_byte_length + header_length
+
+    def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
+        for c in self.codecs:
+            chunk_spec = c.resolve_metadata(chunk_spec)
+        return chunk_spec
+
+    def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
+        transpose = self.make_transpose(array_spec.ndim).evolve_from_array_spec(
+            array_spec
+        )
+        endian = self.make_bytes().evolve_from_array_spec(array_spec)
+
+        codecs: CodecTuple
+        match len(self.codecs):
+            case 2:
+                codecs = (transpose, endian)
+            case 3:
+                compressor: BytesBytesCodec = self.codecs[2].evolve_from_array_spec(  # type:ignore
+                    array_spec
+                )
+                codecs = (transpose, endian, compressor)
+            case _:
+                raise ValueError("unsupported number of codecs")
+
+        return type(self)(codecs=codecs)
+
+    def validate(
+        self,
+        *,
+        shape: tuple[int, ...],
+        dtype: ZDType[TBaseDType, TBaseScalar],
+        chunk_grid: ChunkGrid,
+    ) -> None:
+        expected_ndim = len(self.codecs[0].order)
+        if len(shape) != expected_ndim:
+            raise ValueError(f"array is {len(shape)}D, codec is {expected_ndim}D")
+        if dtype._zarr_v3_name not in COMPATIBLE_DATA_TYPES:
+            raise ValueError(f"N5 does not support data type {dtype._zarr_v3_name}")
+
+        return super().validate(shape=shape, dtype=dtype, chunk_grid=chunk_grid)
+
+    async def _decode_single(
+        self, chunk_data: Buffer, chunk_spec: ArraySpec
+    ) -> NDBuffer:
+        b = chunk_data.as_buffer_like()
+        header = N5BlockHeader.from_bytes(b)
+        offset = header.size()
+
+        body_buf = chunk_data[offset:]
+        body_nd = chunk_spec.prototype.nd_buffer.empty(
+            header.shape, chunk_spec.dtype.to_native_dtype(), chunk_spec.order
+        )
+        if header.shape == chunk_spec.shape:
+            body_spec = chunk_spec
+            all_eq = True
+        else:
+            body_spec = ArraySpec(
+                header.shape,
+                chunk_spec.dtype,
+                chunk_spec.fill_value,
+                chunk_spec.config,
+                chunk_spec.prototype,
+            )
+            all_eq = False
+        maybe_body_nd, *_ = await self.codec_pipeline.decode([(body_buf, body_spec)])
+        # TODO: use codec_pipeline.read() instead; this should avoid the copy for truncated-block cases
+        if maybe_body_nd is None:
+            raise RuntimeError("unexpected nullish buffer")
+        else:
+            body_nd = maybe_body_nd
+
+        if all_eq:
+            # don't need to truncate or pad
+            return body_nd
+
+        # whether we can get the chunk we want by trimming down the N5 block body
+        can_trim = True
+
+        min_shape = []
+        slice_lst = []
+        for hs, cs in zip(header.shape, chunk_spec.shape):
+            if cs > hs:
+                # requested chunk is larger than the N5 block in some dimension
+                can_trim = False
+            min_len = min(hs, cs)
+            min_shape.append(min_len)
+            slice_lst.append(slice(0, min_len))
+
+        slicing = tuple(slice_lst)
+
+        if can_trim:
+            return body_nd[slicing]
+
+        out = chunk_spec.prototype.nd_buffer.create(
+            shape=chunk_spec.shape,
+            dtype=chunk_spec.dtype.to_native_dtype(),
+            order=chunk_spec.order,
+            fill_value=chunk_spec.fill_value,
+        )
+        out[slicing] = body_nd[slicing]
+        return out
+
+    # async def _encode_single(
+    #     self, chunk_data: NDBuffer, chunk_spec: ArraySpec
+    # ) -> Buffer | None:
+    #     header = N5BlockHeader(N5Mode.DEFAULT, chunk_spec.shape)
+    #     for c in self.codecs:
+    #         chunk_data = c._encode_single(chunk_data, chunk_spec)  # type:ignore
+    #         chunk_spec = c.resolve_metadata(chunk_spec)
+
+    #     buf: Buffer = chunk_data  # type: ignore
+
+    #     bio = BytesIO()
+    #     bio.write(header.to_bytes())
+    #     # TODO: avoid this copy?
+    #     bio.write(buf.as_buffer_like())
+    #     return Buffer.from_bytes(bio.getbuffer())
+
+    @classmethod
+    def from_dict(
+        cls,
+        data: dict[str, JSON],
+    ) -> Self:
+        _, configuration_parsed = parse_named_configuration(
+            data, N5_DEFAULT_NAME, require_configuration=True
+        )
+
+        return cls(**configuration_parsed)  # type: ignore[arg-type]
+
+    def to_dict(
+        self,
+    ) -> dict[str, JSON]:
+        return {
+            "name": N5_DEFAULT_NAME,
+            "configuration": {"codecs": [c.to_dict() for c in self.codecs]},
+        }

zarr_n5-0.1.0/src/zarr_n5/constants.py

@@ -0,0 +1,7 @@
+"""Useful constant values."""
+
+N5_METADATA_KEY = "attributes.json"
+"""Object name for N5 metadata files."""
+
+ZARR_V3_METADATA_KEY = "zarr.json"
+"""Object name for Zarr v3 metadata files."""

zarr_n5-0.1.0/src/zarr_n5/convert.py

@@ -0,0 +1,103 @@
+import asyncio
+import logging
+
+import zarr.api.asynchronous as zarr_async
+from zarr.abc.store import Store
+from zarr.core.metadata.io import save_metadata
+from zarr.storage import StoreLike, StorePath
+from zarr.storage._common import make_store
+from zarr.core.group import AsyncGroup
+from zarr.core.array import AsyncArray
+from .storage import ImplicitGroupWrapperStore, N5WrapperStore
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["N5ToZarr", "convert_hierarchy"]
+
+DEFAULT_TASKS = 10
+
+
+class Finished:
+    pass
+
+
+class N5ToZarr:
+    def __init__(self, store: Store, infer_groups: bool = True) -> None:
+        self.inner_store = store
+
+        self.n5_store: Store
+        if infer_groups:
+            self.n5_store = ImplicitGroupWrapperStore(N5WrapperStore(store))
+        else:
+            self.n5_store = N5WrapperStore(store)
+
+        self.queue: asyncio.Queue[AsyncArray | AsyncGroup | Finished] = asyncio.Queue()
+
+    async def convert_hierarchy(
+        self, path: str = "", max_depth: int | None = -1, n_tasks=10
+    ):
+        member = await zarr_async.open(store=self.n5_store, path=path)
+        total = await self.convert_member(member)
+        if total == 0:
+            return total
+        if isinstance(member, AsyncArray):
+            return total
+        if max_depth is not None and max_depth >= 0:
+            return total
+
+        new_depth = None if max_depth is None else max_depth - 1
+        tasks = [self._spawn_worker() for _ in range(n_tasks)]
+        total = 0
+        async for _, child in member.members(new_depth):
+            await self.queue.put(child)
+            total += 1
+        logger.info("Enqueued %s nodes", total)
+
+        await self.queue.put(Finished())
+        count = sum(await asyncio.gather(*tasks))
+        logger.info("Converted %s nodes", count)
+        return count
+
+    def _spawn_worker(self, name: str | None = None):
+        """Schedule a task for execution wrapping a worker function."""
+        return asyncio.create_task(self._worker(), name=name)
+
+    async def _worker(self):
+        """Create a worker which reads Zarr nodes from the queue and processes them."""
+        total = 0
+        while True:
+            value = await self.queue.get()
+            if isinstance(value, Finished):
+                await self.queue.put(value)
+                return total
+            total += await self.convert_member(value)
+            self.queue.task_done()
+
+    async def convert_member(self, member: AsyncArray | AsyncGroup) -> int:
+        """Returns 0 or 1 for whether the node was skipped or converted."""
+        try:
+            _ = await zarr_async.open(
+                store=self.inner_store, mode="r", path=member.path
+            )
+            logger.info("Found existing zarr node at %s, ignoring", member.path)
+            return 0
+        except Exception:
+            pass
+
+        await save_metadata(
+            StorePath(store=self.inner_store, path=member.path), member.metadata, False
+        )
+        logger.info("Converted N5 entry %s to Zarr", member.path)
+        return 1
+
+
+async def convert_hierarchy(
+    store: StoreLike,
+    path: str = "",
+    infer_groups: bool = True,
+    max_depth=None,
+    n_tasks=DEFAULT_TASKS,
+) -> int:
+    inner_store = await make_store(store, mode="r+")
+    converter = N5ToZarr(inner_store, infer_groups)
+    return await converter.convert_hierarchy(path, max_depth=max_depth, n_tasks=n_tasks)

zarr_n5-0.1.0/src/zarr_n5/metadata.py

@@ -0,0 +1,194 @@
+"""
+Utilities for parsing, representing, and converting N5 metadata.
+"""
+
+from __future__ import annotations
+from copy import deepcopy
+import itertools
+from typing import Any, TYPE_CHECKING, Self
+from zarr.core.group import GroupMetadata
+from zarr.core.metadata.v3 import ArrayV3Metadata
+from zarr.core.dtype import ZDType
+from zarr.core import dtype as zdt
+from zarr.core.chunk_grids import RegularChunkGrid
+from zarr.core.chunk_key_encodings import V2ChunkKeyEncoding
+from zarr.abc.codec import BytesBytesCodec
+from zarr.codecs import blosc
+from zarr.codecs import GzipCodec, ZstdCodec
+
+from .util import N5Mode
+
+if TYPE_CHECKING:
+    from typing import Self
+    from zarr.core.common import JSON
+
+__all__ = ["N5GroupMetadata", "N5ArrayMetadata", "COMPATIBLE_DATA_TYPES"]
+
+COMPATIBLE_DATA_TYPES: dict[str, tuple[ZDType, int]] = {
+    "uint8": (zdt.UInt8(), 1),
+    "uint16": (zdt.UInt16(), 2),
+    "uint32": (zdt.UInt32(), 4),
+    "uint64": (zdt.UInt64(), 8),
+    "int8": (zdt.Int8(), 1),
+    "int16": (zdt.Int16(), 2),
+    "int32": (zdt.Int32(), 4),
+    "int64": (zdt.Int64(), 8),
+    "float32": (zdt.Float32(), 4),
+    "float64": (zdt.Float64(), 8),
+}
+"""Data types which exist in both Zarr and N5.
+
+Maps to the Zarr data type and item size."""
+
+
+class N5GroupMetadata:
+    def __init__(
+        self, n5: str | None = None, attrs: dict[str, JSON] | None = None
+    ) -> None:
+        self.n5: str | None = n5
+        self.attributes: dict[str, Any] = attrs or dict()
+
+    def to_jso(self) -> dict[str, JSON]:
+        out = deepcopy(self.attributes)
+        if self.n5 is not None:
+            out["n5"] = self.n5
+        return out
+
+    def is_root(self):
+        return self.n5 is not None
+
+    @classmethod
+    def from_jso(cls, jso: dict[str, JSON]) -> Self:
+        n5 = jso.pop("n5", None)
+        if n5 is not None and not isinstance(n5, str):
+            raise ValueError("n5 attribute is not a string")
+        return cls(n5, jso)
+
+    def to_zarr(self):
+        attrs = deepcopy(self.attributes)
+        attrs["_n5"] = self.to_jso()
+        return GroupMetadata(attrs)
+
+
+class N5ArrayMetadata(N5GroupMetadata):
+    def __init__(
+        self,
+        dimensions: list[int],
+        block_size: list[int],
+        data_type: str,
+        compression: dict[str, Any],
+        n5: str | None = None,
+        attrs: dict[str, JSON] | None = None,
+    ):
+        super().__init__(n5, attrs)
+        if len(dimensions) != len(block_size):
+            raise ValueError(
+                f"dimensions {dimensions} and block size {block_size} must have same dimensionality"
+            )
+
+        if any(not is_nonzero_int(s) for s in itertools.chain(dimensions, block_size)):
+            raise ValueError("dimensions and block size must be positive integers")
+
+        self.dimensions = dimensions
+        self.block_size = block_size
+        self.data_type = data_type
+        ctype = compression.get("type")
+        if not isinstance(ctype, str):
+            raise ValueError(f"compression must have a string type, got {ctype}")
+        self.compression = compression
+
+    def to_jso(self) -> dict[str, JSON]:
+        jso = super().to_jso()
+        jso["dimensions"] = self.dimensions
+        jso["blockSize"] = self.block_size
+        jso["dataType"] = self.data_type
+        jso["compression"] = self.compression
+        return jso
+
+    @classmethod
+    def from_group(cls, grp: N5GroupMetadata) -> Self:
+        attrs = deepcopy(grp.attributes)
+        dimensions = attrs.pop("dimensions")
+        block_size = attrs.pop("blockSize")
+        data_type = attrs.pop("dataType")
+        compression = attrs.pop("compression")
+        return cls(dimensions, block_size, data_type, compression, grp.n5, attrs)
+
+    @classmethod
+    def from_jso(cls, jso: dict[str, JSON]) -> Self:
+        grp = super().from_jso(jso)
+        return cls.from_group(grp)
+
+    def to_zarr(self, mode: N5Mode = N5Mode.DEFAULT):
+        from .codec.default import N5DefaultCodec
+
+        if mode != N5Mode.DEFAULT:
+            raise NotImplementedError("Only default-mode N5 is supported")
+        compressor = self._to_zarr_codec()
+        attrs = deepcopy(self.attributes)
+        attrs["_n5"] = self.to_jso()
+        return ArrayV3Metadata(
+            shape=self.dimensions,
+            data_type=COMPATIBLE_DATA_TYPES[self.data_type][0],
+            chunk_grid=RegularChunkGrid(chunk_shape=self.block_size),
+            chunk_key_encoding=V2ChunkKeyEncoding("/"),
+            fill_value=0,
+            dimension_names=None,
+            codecs=[
+                N5DefaultCodec.from_compressor(
+                    len(self.dimensions),
+                    compressor,
+                ),
+            ],
+            attributes=attrs,
+        )
+
+    def _to_zarr_codec(self) -> BytesBytesCodec | None:
+        tp = self.compression.get("type")
+        match tp:
+            case "raw":
+                return None
+            case "blosc":
+                item_size = COMPATIBLE_DATA_TYPES[self.data_type][1]
+                return parse_blosc(self.compression, item_size)
+            case "gzip":
+                return parse_gzip(self.compression)
+            case "zstd":
+                return parse_zstd(self.compression)
+            case _:
+                raise ValueError(f"unsupported codec with type {tp}")
+
+
+def is_nonzero_int(n) -> bool:
+    if not isinstance(n, int):
+        return False
+    return n > 0
+
+
+def parse_blosc(d: dict[str, JSON], typesize: int | None) -> blosc.BloscCodec:
+    cname = d.get("cname", "blosclz")
+    clevel = d.get("clevel", 6)
+    blocksize = d.get("blocksize", 0)
+    shuffle_int = d.get("shuffle", 0)
+    shuffle = blosc.BloscShuffle.from_int(shuffle_int)  # type: ignore
+    return blosc.BloscCodec(
+        typesize=typesize,
+        cname=cname,  # type: ignore
+        clevel=clevel,  # type:ignore
+        blocksize=blocksize,  # type:ignore
+        shuffle=shuffle,
+    )
+
+
+def parse_gzip(d: dict[str, JSON]) -> GzipCodec:
+    level = d.get("level", -1)
+    if level == -1:
+        return GzipCodec()
+    else:
+        level = int(level)  # type: ignore
+    return GzipCodec(level=level)
+
+
+def parse_zstd(d: dict[str, JSON]) -> ZstdCodec:
+    level = d.get("level", 3)
+    return ZstdCodec(level=level)  # type: ignore

zarr_n5-0.1.0/src/zarr_n5/storage/__init__.py

@@ -0,0 +1,8 @@
+"""
+Storage wrappers for N5 data.
+"""
+
+from .n5 import N5WrapperStore
+from .implicit import ImplicitGroupWrapperStore
+
+__all__ = ["N5WrapperStore", "ImplicitGroupWrapperStore"]

zarr_n5-0.1.0/src/zarr_n5/storage/implicit.py

@@ -0,0 +1,78 @@
+"""
+Module containing `ImplicitGroupWrapperStore`,
+for inferring groups with missing metadata.
+"""
+
+from typing import Final
+from collections.abc import Iterable
+import json
+
+from zarr.abc.store import (
+    Store,
+    ByteRequest,
+)
+from zarr.storage import WrapperStore
+from zarr.core.group import GroupMetadata
+from zarr.core.buffer import BufferPrototype, Buffer
+
+from ..util import slice_buf, is_zarr3_metadata
+
+__all__ = ["ImplicitGroupWrapperStore"]
+
+
+def make_implicit_group_bytes() -> bytes:
+    g = GroupMetadata()
+    g.attributes["_implicit"] = True
+
+    return json.dumps(g.to_dict()).encode()
+
+
+IMPLICIT_GROUP_BYTES: Final[bytes] = make_implicit_group_bytes()
+
+
+class ImplicitGroupWrapperStore[T: Store](WrapperStore):
+    """A store which supplies empty group metadata documents if they do not exist.
+
+    Used to replicate N5's behaviour where any directory (or prefix) is a valid group,
+    even when no metadata document exists.
+    Wrap over an `N5WrapperStore`.
+
+    Inferred group metadata's attributes will contain the key/value `"_implicit": true`.
+    """
+
+    _store: T
+
+    async def get(
+        self,
+        key: str,
+        prototype: BufferPrototype,
+        byte_range: ByteRequest | None = None,
+    ) -> Buffer | None:
+        res = await self._store.get(key, prototype, byte_range)
+        if res is not None or not is_zarr3_metadata(key):
+            return res
+
+        b = slice_buf(IMPLICIT_GROUP_BYTES, byte_range)
+        return prototype.buffer.from_bytes(b)
+
+    async def get_partial_values(
+        self,
+        prototype: BufferPrototype,
+        key_ranges: Iterable[tuple[str, ByteRequest | None]],
+    ) -> list[Buffer | None]:
+        key_ranges = list(key_ranges)
+        reses = await super().get_partial_values(prototype, key_ranges)
+        out = []
+        for (key, byte_range), res in zip(key_ranges, reses):
+            if res is None and is_zarr3_metadata(key):
+                res = prototype.buffer.from_bytes(
+                    slice_buf(IMPLICIT_GROUP_BYTES, byte_range)
+                )
+            out.append(res)
+
+        return out
+
+    async def exists(self, key: str) -> bool:
+        if is_zarr3_metadata(key):
+            return True
+        return await super().exists(key)

zarr_n5-0.1.0/src/zarr_n5/storage/n5.py

@@ -0,0 +1,159 @@
+"""
+Module containing `N5WrapperStore`,
+for silently converting N5 nodes to Zarr nodes.
+"""
+
+from collections import defaultdict
+from collections.abc import AsyncIterator, Iterable
+from zarr.storage import WrapperStore
+from zarr.abc.store import (
+    Store,
+    ByteRequest,
+)
+from zarr.core.buffer import Buffer, BufferPrototype
+import json
+import asyncio
+
+from ..constants import N5_METADATA_KEY, ZARR_V3_METADATA_KEY
+from ..metadata import N5GroupMetadata, N5ArrayMetadata
+from ..util import slice_buf, is_zarr3_metadata, N5Mode
+
+
+class N5WrapperStore[T: Store](WrapperStore):
+    """A read-only store for opening N5 hierarchies.
+
+    Requests for Zarr metadata documents are redirected to N5 attributes,
+    and Zarr metadata calculated on the fly.
+
+    Note that N5 attributes can be omitted in groups.
+    You may want to wrap this in an `ImplicitGroupWrapperStore` to replicate that behaviour.
+
+    Only compatible with DEFAULT-mode N5 arrays.
+    """
+
+    _store: T
+
+    def intercept_metadata(self, key: str) -> None | str:
+        """If the given key is for Zarr v3 metadata, return the key for N5 metadata in the equivalent node.
+
+        Otherwise, return None.
+        """
+        if "/" in key:
+            pref, fname = key.rsplit("/", 1)
+        else:
+            pref = None
+            fname = key
+
+        if fname != ZARR_V3_METADATA_KEY:
+            return None
+
+        if pref is None:
+            k2 = N5_METADATA_KEY
+        else:
+            k2 = f"{pref}/{N5_METADATA_KEY}"
+
+        return k2
+
+    async def get(
+        self,
+        key: str,
+        prototype: BufferPrototype,
+        byte_range: ByteRequest | None = None,
+    ) -> Buffer | None:
+        k2 = self.intercept_metadata(key)
+        if k2 is None:
+            return await self._store.get(key, prototype, byte_range)
+
+        b = await self._store.get(k2, prototype)
+
+        if b is None:
+            return None
+
+        d = json.loads(b.to_bytes())
+        n5_meta = N5GroupMetadata.from_jso(d)
+        try:
+            n5_meta = N5ArrayMetadata.from_group(n5_meta)
+            out_d = n5_meta.to_zarr(N5Mode.DEFAULT)
+        except KeyError:
+            out_d = n5_meta.to_zarr()
+
+        b2 = json.dumps(out_d.to_dict()).encode()
+        b2 = slice_buf(b2, byte_range)
+
+        return prototype.buffer.from_bytes(b2)
+
+    async def get_partial_values(
+        self,
+        prototype: BufferPrototype,
+        key_ranges: Iterable[tuple[str, ByteRequest | None]],
+    ) -> list[Buffer | None]:
+
+        # Split the key ranges into metadata requests and other (chunk) requests.
+        # We always need to read the whole N5 metadata file
+        # to convert it into Zarr v3 metadata before slicing it,
+        # so this prevents reading it multiple times.
+        meta_reqs: defaultdict[str, list[tuple[int, ByteRequest | None]]] = defaultdict(
+            list
+        )
+        other_reqs: list[tuple[int, tuple[str, ByteRequest | None]]] = []
+        count = 0
+        for idx, (key, byte_range) in enumerate(key_ranges):
+            if is_zarr3_metadata(key):
+                meta_reqs[key].append((idx, byte_range))
+            else:
+                other_reqs.append((idx, (key, byte_range)))
+            count += 1
+
+        other_reqs_fut = self._store.get_partial_values(
+            prototype, (tup[1] for tup in other_reqs)
+        )
+        meta_req_list = list(meta_reqs.items())
+        meta_reqs_fut = asyncio.gather(
+            *(self.get(k, prototype) for k, _ in meta_req_list)
+        )
+        # Gather all requests to run concurrently
+        other_res, meta_res = await asyncio.gather(other_reqs_fut, meta_reqs_fut)
+        out: list[None | Buffer] = [None for _ in range(count)]
+
+        # Slice and insert the metadata responses into the pre-allocated output list
+        for res, (_, meta_req) in zip(meta_res, meta_req_list):
+            if res is None:
+                continue
+            blike = res.as_buffer_like()
+            for idx, byte_range in meta_req:
+                out[idx] = Buffer.from_bytes(slice_buf(blike, byte_range))
+
+        # Insert the non-metadata responses into the output;
+        # these are already sliced by the underlying store.
+        for res, (idx, _) in zip(other_res, other_reqs):
+            out[idx] = res
+
+        return out
+
+    async def exists(self, key: str) -> bool:
+        k2 = self.intercept_metadata(key)
+        return await self._store.exists(k2 or key)
+
+    @property
+    def supports_writes(self) -> bool:
+        return False
+
+    @property
+    def supports_deletes(self) -> bool:
+        return False
+
+    async def delete(self, key: str) -> None:
+        raise NotImplementedError
+
+    @property
+    def supports_listing(self) -> bool:
+        return self._store.supports_listing
+
+    def list(self) -> AsyncIterator[str]:
+        return self._store.list()
+
+    def list_prefix(self, prefix: str) -> AsyncIterator[str]:
+        return self._store.list_prefix(prefix)
+
+    def list_dir(self, prefix: str) -> AsyncIterator[str]:
+        return self._store.list_dir(prefix)

zarr_n5-0.1.0/src/zarr_n5/util.py

@@ -0,0 +1,115 @@
+"""
+General utilities.
+"""
+
+from zarr.abc.store import (
+    ByteRequest,
+    RangeByteRequest,
+    OffsetByteRequest,
+    SuffixByteRequest,
+)
+from .constants import ZARR_V3_METADATA_KEY
+from dataclasses import dataclass
+from enum import IntEnum
+from typing import Self, Any
+import struct
+
+__all__ = ["N5Mode", "N5BlockHeader"]
+
+
+class N5Mode(IntEnum):
+    """N5 block mode"""
+
+    DEFAULT = 0
+    VARLENGTH = 1
+    OBJECT = 2
+
+
+@dataclass
+class N5BlockHeader:
+    """Parsed representation of the N5 block header."""
+
+    mode: N5Mode
+    """Stored as >u16"""
+
+    shape: tuple[int, ...]
+    """Length stored as >u16, elements stored as >u32"""
+
+    num_elem: int | None = None
+    """Stored as >u32 if mode == VARLENGTH"""
+
+    def __post_init__(self):
+        if self.num_elem is not None and self.mode != N5Mode.VARLENGTH:
+            raise ValueError("num_elem must be None if mode is not VARLENGTH")
+
+    @classmethod
+    def calc_size(cls, ndim: int, is_varlength: bool = False) -> int:
+        """Calculate the number of bytes in an N5 block header."""
+        base = 2 + 2 + 4 * ndim
+        if is_varlength:
+            base += 4
+        return base
+
+    def size(self) -> int:
+        """Determine the number of bytes this header will take."""
+        return self.calc_size(len(self.shape), self.mode == N5Mode.VARLENGTH)
+
+    @classmethod
+    def from_bytes(cls, b: bytes) -> Self:
+        p = StructParser(b, ">")
+        mode_num, ndim = p.unpack("HH")
+        mode = N5Mode(mode_num)
+
+        shape = p.unpack("I" * ndim)
+
+        if mode == N5Mode.VARLENGTH:
+            numel = p.unpack("I")[0]
+        else:
+            numel = None
+
+        return cls(mode=mode, shape=shape, num_elem=numel)
+
+    @property
+    def ndim(self):
+        return len(self.shape)
+
+    def to_bytes(self) -> bytes:
+        fmt = ">HH" + "I" * self.ndim
+        args = [self.mode, self.ndim, *self.shape]
+        if self.num_elem is not None:
+            fmt += "I"
+            args.append(self.num_elem)
+        return struct.pack(fmt, *args)
+
+
+class StructParser:
+    def __init__(self, buf: bytes, endian: str = "") -> None:
+        self.endian = endian
+        self.buf = buf
+        self.offset = 0
+
+    def unpack(self, fmt: str) -> tuple[Any, ...]:
+        fmt = self.endian + fmt
+        sz = struct.calcsize(fmt)
+        out = struct.unpack(fmt, self.buf[self.offset : self.offset + sz])
+        self.offset += sz
+        return out
+
+
+def slice_buf(b: bytes, byte_range: ByteRequest | None = None) -> bytes:
+    """Optionally slice a byte buffer."""
+    if byte_range is None:
+        return b
+    elif isinstance(byte_range, RangeByteRequest):
+        b = b[byte_range.start : byte_range.end]
+    elif isinstance(byte_range, OffsetByteRequest):
+        b = b[byte_range.offset :]
+    elif isinstance(byte_range, SuffixByteRequest):
+        b = b[-byte_range.suffix :]
+
+    raise TypeError(f"byte_range argument has unknown type {type(byte_range)}")
+
+
+def is_zarr3_metadata(key: str):
+    """Whether a key belongs to a Zarr v3 metadata object."""
+    return key.split("/")[-1] == ZARR_V3_METADATA_KEY