micro-reader 0.0.1__py3-none-any.whl

micro_reader/__init__.py

@@ -0,0 +1,23 @@
+"""micro-reader: read exactly the rectangle you ask for.
+
+>>> import micro_reader
+>>> with micro_reader.open("image.mrc") as f:
+...     block = f.images[0].read(z=0, y=slice(0, 512))
+"""
+from ._base import INDEX_NAMES, SAMPLED_INDICES, File, Image, ReadStats
+from ._metadata import Channel, Metadata, Quantity
+from ._errors import CorruptFileError, MicroReaderError, UnsupportedFormatError
+from ._memory import MemoryLimitError, memory_limit, set_memory_limit
+from ._pool import decode_threads, set_decode_threads
+from ._registry import BACKENDS, open  # noqa: A004 - the public entry point
+from .backends.czi import DamagedFileWarning
+
+__version__ = "0.0.1"
+
+__all__ = [
+    "open", "File", "Image", "INDEX_NAMES", "SAMPLED_INDICES", "ReadStats", "BACKENDS",
+    "Metadata", "Channel", "Quantity",
+    "decode_threads", "set_decode_threads", "memory_limit", "set_memory_limit",
+    "MicroReaderError", "UnsupportedFormatError", "CorruptFileError", "MemoryLimitError", "DamagedFileWarning",
+    "__version__",
+]

micro_reader/_axes.py

@@ -0,0 +1,196 @@
+"""An image presented with chosen axes (``Image.as_axes``, ``Image.squeeze``,
+``open(axes=..., squeeze=...)``).
+
+Arrays have only ``t c z y x`` and ``s`` (RGB samples); every other
+dimension a file has is an image index (docs/AXES.md).  Backends report the
+axes an image has: formats with an order of their own (TIFF, MRC) in that
+order, the others in ``tczyx``.  An axes image rearranges them without
+guessing:
+
+- ``axes=None`` keeps the image's order; ``"canonical"`` gives ``t c z y x``,
+  then ``s``; any other string is the exact axes wanted;
+- axes the file lacks are inserted with size 1;
+- RGB samples ``s`` stay a trailing ``s`` when asked for, or are folded into
+  the channels with ``samples="channels"`` (channel ``k`` = sample ``k % S``
+  of channel ``k // S``, as eubi-bridge stores them);
+- ``squeeze=True`` drops every size-1 axis except ``y`` and ``x``;
+  ``squeeze="tz"`` drops just those (each must have size 1);
+- an axis the requested ones cannot hold is an error that names it.
+
+Reads stay exact: each request is translated into the native axes, read
+there, and rearranged; the native image stays available as ``.source``.
+"""
+from __future__ import annotations
+
+from typing import Optional, Union
+
+import numpy as np
+
+from ._base import Image
+
+SAMPLE_POLICIES = ("keep", "channels")
+#: the array axes, outermost first; ``s`` (RGB samples) after them
+STANDARD = "tczyx"
+CANONICAL = STANDARD + "s"
+#: never dropped by ``squeeze=True``: a plane stays a plane
+KEPT_BY_SQUEEZE = "yx"
+
+
+class AxesImage(Image):
+    """*source* presented with *axes* (see the module docstring)."""
+
+    def __init__(self, source: Image, axes: Optional[str] = None, *, samples: str = "keep",
+                 squeeze: Union[bool, str] = False, _select: Optional[dict] = None):
+        if samples not in SAMPLE_POLICIES:
+            raise ValueError(f"samples={samples!r}: one of {SAMPLE_POLICIES}")
+        names = list(source.axes)
+        #: native size-1 axes left out (squeezed): read at index 0
+        select = dict(_select or {})
+        axes = self._resolve(source, axes, names, select, samples)
+        self.source = source
+        self._names, self._select = names, select
+        self._samples_policy = samples
+        sizes = dict(zip(names, source.shape))
+        # RGB samples folded into the channels
+        self._fold = "s" in names and "s" not in axes and "s" not in select \
+            and samples == "channels"
+        if self._fold:
+            sizes["c"] = sizes.get("c", 1) * sizes.pop("s")
+        left = [a for a in names if a not in select and a not in axes
+                and not (self._fold and a == "s") and not (self._fold and a == "c")]
+        if left:
+            hint = ("; samples='channels' folds RGB samples into the channels"
+                    if "s" in left else "")
+            raise ValueError(f"{source.name!r} has axes {''.join(left)!r} that {axes!r} cannot "
+                             f"hold{hint}")
+        if self._fold and "c" not in axes:
+            raise ValueError(f"samples='channels' needs 'c' in the axes (got {axes!r})")
+        if squeeze:
+            axes = self._squeeze(axes, sizes, squeeze, names, select, source.name)
+        shape = [sizes.get(a, 1) for a in axes]
+        self._samples = source.shape[names.index("s")] if self._fold else 1
+        unit = None
+        if source.read_unit is not None:
+            native_unit = dict(zip(names, source.read_unit))
+            if self._fold:
+                native_unit["c"] = native_unit.get("c", 1) * native_unit.pop("s")
+            unit = [native_unit.get(a, 1) for a in axes]
+        super().__init__(source._position, axes, shape, source.dtype, name=source.name,
+                         read_unit=unit, index=source.index)
+
+    @staticmethod
+    def _resolve(source, axes, names, select, samples) -> str:
+        """The axes string asked for: None, "canonical" or explicit letters."""
+        free = [a for a in names if a not in select]
+        fold = samples == "channels" and "s" in free
+        if axes is None:                                     # the image's own order
+            if fold:
+                free.remove("s")
+                if "c" not in free:                          # RGB without channels: c
+                    at = next((i for i, a in enumerate(free) if a in "zyx"), len(free))
+                    free.insert(at, "c")                     # where the canonical order has it
+            return "".join(free)
+        axes = axes.lower()
+        if axes == "canonical":
+            return STANDARD + ("s" if "s" in free and not fold else "")
+        if len(set(axes)) != len(axes) or not axes.isalpha():
+            raise ValueError(f"axes {axes!r}: one letter per axis, each once")
+        stray = sorted(set(axes) - set(CANONICAL))
+        if stray:
+            raise ValueError(f"axes {axes!r}: {stray} are not array axes (arrays have "
+                             f"{CANONICAL!r}; other dimensions are image indices)")
+        return axes
+
+    def _squeeze(self, axes, sizes, squeeze, names, select, name) -> str:
+        """*axes* without the squeezed ones; squeezed native axes are read at
+        index 0 (*select* is updated in place)."""
+        if squeeze is True:
+            drop = [a for a in axes if sizes.get(a, 1) == 1 and a not in KEPT_BY_SQUEEZE]
+        else:
+            drop = list(squeeze.lower())
+            for a in drop:
+                if a not in axes:
+                    raise ValueError(f"squeeze {a!r}: {name!r} has axes {axes!r}")
+                if sizes.get(a, 1) != 1:
+                    raise ValueError(f"squeeze {a!r}: axis {a!r} of {name!r} has "
+                                     f"{sizes[a]}, not 1")
+        for a in drop:
+            if a in names and not (self._fold and a == "c"):
+                select[a] = 0
+        return "".join(a for a in axes if a not in drop)
+
+    def _read_block(self, block):
+        names, select, axes = self._names, self._select, self.axes
+        want = dict(zip(axes, block))
+        S = self._samples
+        if self._fold:                          # folded channels k0:k1 -> channels, all samples
+            k = want.get("c", slice(0, 1))
+            c0, c1 = k.start // S, (k.stop - 1) // S + 1
+        native_key = []
+        for a, size in zip(names, self.source.shape):
+            if a in select:
+                native_key.append(slice(select[a], select[a] + 1))
+            elif self._fold and a == "s":
+                native_key.append(slice(0, S))
+            elif self._fold and a == "c":
+                native_key.append(slice(c0, c1))
+            else:
+                native_key.append(want[a])
+        data, read = self.source._read_block(tuple(native_key))
+        order = list(names)
+        # drop squeezed axes
+        for a in select:
+            data = data.squeeze(axis=order.index(a))
+            order.remove(a)
+        if self._fold:
+            if "c" not in order:                  # RGB without channels: one channel
+                data = np.expand_dims(data, order.index("s"))
+                order.insert(order.index("s"), "c")
+            ci, si = order.index("c"), order.index("s")
+            data = np.moveaxis(data, si, ci + 1 if si > ci else ci)
+            order.remove("s")
+            ci = order.index("c")
+            shape = list(data.shape)
+            shape[ci:ci + 2] = [shape[ci] * shape[ci + 1]]
+            data = data.reshape(shape)
+            k = want.get("c", slice(0, 1))
+            index = [slice(None)] * data.ndim
+            index[ci] = slice(k.start - c0 * S, k.stop - c0 * S)
+            data = data[tuple(index)]
+        data = data.transpose([order.index(a) for a in axes if a in order])
+        for i, a in enumerate(axes):                         # axes the file lacks
+            if a not in order:
+                data = np.expand_dims(data, i)
+        return np.ascontiguousarray(data), read
+
+    @property
+    def metadata(self):
+        """The native image's metadata (channels follow its own c axis)."""
+        return self.source.metadata
+
+    @property
+    def resolutions(self) -> list:
+        """The native image's pyramid, each level with these axes."""
+        levels = self.source.resolutions
+        if len(levels) == 1:
+            return [self]
+        return [self if level is self.source else
+                AxesImage(level, self.axes, samples=self._samples_policy,
+                          _select=dict(self._select))
+                for level in levels]
+
+    def as_axes(self, axes: Optional[str] = None, **policies) -> "AxesImage":
+        """An axes image of the native image with other axes (they do not stack)."""
+        return self.source.as_axes(axes, **policies)
+
+    def squeeze(self, axes: Optional[str] = None) -> "AxesImage":
+        """This axes image without its size-1 axes (all but ``y`` / ``x``, or those
+        in *axes*), rebuilt over the native image."""
+        return AxesImage(self.source, self.axes, samples=self._samples_policy,
+                         squeeze=axes or True, _select=dict(self._select))
+
+    def close(self) -> None:
+        self.source.close()
+
+    def __repr__(self) -> str:
+        return super().__repr__()[:-1] + f" of {self.source.axes!r}>"

micro_reader/_base.py

@@ -0,0 +1,268 @@
+"""The interface every backend implements: a File holding Images.
+
+Each image is one spatial frame of the file; its index says which (scene,
+tile, rotation, view: see docs/AXES.md)."""
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass, field
+from typing import Optional, Sequence
+
+import numpy as np
+
+from ._index import normalise_index
+
+
+@dataclass
+class ReadStats:
+    """What an image has read so far; the over-read check of the test harness.
+
+    ``pixels_returned`` counts the contiguous spans requested (before any step
+    subsampling); ``pixels_read`` what the backend actually had to read or
+    decode.  A backend that reads exact rectangles keeps them equal.
+    """
+    reads: int = 0
+    pixels_returned: int = 0
+    pixels_read: int = 0
+    _lock: threading.Lock = field(default_factory=threading.Lock, repr=False,
+                                  compare=False)
+
+    @property
+    def over_read(self) -> float:
+        return (self.pixels_read / self.pixels_returned
+                if self.pixels_returned else 1.0)
+
+    def add(self, returned: int, read: int) -> None:
+        with self._lock:
+            self.reads += 1
+            self.pixels_returned += returned
+            self.pixels_read += read
+
+    def reset(self) -> None:
+        with self._lock:
+            self.reads = self.pixels_returned = self.pixels_read = 0
+
+
+class Image:
+    """One image of a file (one spatial frame), read by rectangle.
+
+    Backends implement ``_read_block``; everything numpy-like -- integer
+    indices, steps, Ellipsis, the keyword form of ``read`` -- is handled here.
+    Results are always native-endian and never views into file-backed memory.
+    """
+
+    def __init__(self, position: int, axes: str, shape: Sequence[int], dtype,
+                 name: Optional[str] = None,
+                 read_unit: Optional[Sequence[int]] = None,
+                 index: Optional[dict] = None):
+        if len(axes) != len(shape):
+            raise ValueError(f"axes {axes!r} do not match shape {tuple(shape)}")
+        #: place in the file's ``images`` (internal; ``f.images.index(image)``)
+        self._position = int(position)
+        self.axes = axes.lower()
+        self.shape = tuple(int(n) for n in shape)
+        self.dtype = np.dtype(dtype).newbyteorder("=")
+        self.name = name if name is not None else f"image{position}"
+        #: which image of the file: {"scene": .., "tile": .., "rotation": ..,
+        #: "view": .., "resolution": ..} -- only the indices the file has
+        #: ("resolution" only when opened with resolutions="all")
+        self.index = dict(index) if index is not None else {"scene": self._position}
+        self._resolutions: Optional[list] = None
+        self._metadata = None           # backends set a _metadata.Metadata
+        #: None: any rectangle is read exactly.  Otherwise the smallest block
+        #: the backend reads at once (e.g. a compressed tile or strip).
+        self.read_unit = None if read_unit is None else tuple(int(n) for n in read_unit)
+        self.stats = ReadStats()
+
+    @property
+    def ndim(self) -> int:
+        return len(self.shape)
+
+    @property
+    def metadata(self):
+        """Pixel and channel metadata (docs/METADATA.md): pixel sizes, stage
+        position, channels (name, colour, wavelengths), essential axis
+        values.  What the file does not say is absent / None.  Read on first
+        access; a resolution level's is level 0's with its pixel sizes scaled
+        by the shape ratio."""
+        if self._metadata is None:
+            from ._metadata import Metadata
+            base = self._resolutions[0] if self._resolutions else self
+            if base is not self:
+                factors = {a: base.shape[base.axes.index(a)] / self.shape[self.axes.index(a)]
+                           for a in "xyz" if a in self.axes and a in base.axes}
+                self._metadata = base.metadata.scaled(factors)
+            else:
+                n = self.shape[self.axes.index("c")] if "c" in self.axes else 1
+                self._metadata = self._read_metadata() or Metadata.unknown(n)
+        return self._metadata
+
+    def _read_metadata(self):
+        """This image's ``_metadata.Metadata`` from the file, or None when the
+        backend reads none (backends override it)."""
+        return None
+
+    @property
+    def is_pyramidal(self) -> bool:
+        """Whether the file stores lower resolution levels of this image
+        (``resolutions`` holds more than the image itself)."""
+        return len(self.resolutions) > 1
+
+    @property
+    def resolutions(self) -> list:
+        """This image's pyramid, full resolution first: one image per
+        resolution level the file stores (just this image without levels)."""
+        return list(self._resolutions) if self._resolutions else [self]
+
+    @property
+    def nbytes(self) -> int:
+        return int(np.prod(self.shape)) * self.dtype.itemsize
+
+    def __getitem__(self, key) -> np.ndarray:
+        spans, squeeze, steps = normalise_index(key, self.shape)
+        block = tuple(slice(a, b) for a, b in spans)
+        requested = int(np.prod([b - a for a, b in spans]))
+        if requested == 0:
+            out = np.empty(tuple(b - a for a, b in spans), dtype=self.dtype)
+            pixels_read = 0
+        else:
+            out, pixels_read = self._read_block(block)
+        self.stats.add(requested, pixels_read)
+        if any(step != 1 for step in steps):
+            out = out[tuple(slice(None, None, step) for step in steps)]
+        if squeeze:
+            out = out.squeeze(axis=squeeze)
+        return out
+
+    def read(self, **selection) -> np.ndarray:
+        """Keyword form: ``image.read(z=3, y=slice(0, 512))``; omitted axes: all."""
+        unknown = set(selection) - set(self.axes)
+        if unknown:
+            raise KeyError(f"unknown axes {sorted(unknown)}; this image has {self.axes!r} "
+                           "(open(path, axes='canonical') or image.as_axes('canonical') "
+                           "adds the missing standard axes with size 1)")
+        return self[tuple(selection.get(ax, slice(None)) for ax in self.axes)]
+
+    def as_axes(self, axes: Optional[str] = None, *, samples: str = "keep",
+                squeeze=False) -> "Image":
+        """This image presented with *axes*: ``None`` keeps its order,
+        ``"canonical"`` is ``tczyx`` (+ ``s``), or explicit letters such as
+        ``"zyx"``.  Missing axes are inserted with size 1, the others
+        reordered; reads stay exact.  ``samples="channels"`` folds RGB
+        samples into the channels (C x S); ``squeeze=True`` drops size-1 axes
+        except y / x.  An axis *axes* cannot hold is an error that names it.
+        The native image is ``.source`` of the result."""
+        from ._axes import AxesImage
+        return AxesImage(self, axes, samples=samples, squeeze=squeeze)
+
+    def squeeze(self, axes: Optional[str] = None) -> "Image":
+        """This image without its size-1 axes: all but ``y`` / ``x``, or just
+        those in *axes* (each must have size 1).  They also leave ``axes``."""
+        from ._axes import AxesImage
+        return AxesImage(self, None, squeeze=axes or True)
+
+    def _read_block(self, block: tuple[slice, ...]) -> tuple[np.ndarray, int]:
+        """Read contiguous unit-step spans -> (array of exactly that shape,
+        number of pixels actually read or decoded to produce it)."""
+        raise NotImplementedError
+
+    def close(self) -> None:
+        """Release file handles; the image is unusable afterwards."""
+
+    def __repr__(self) -> str:
+        unit = "any rectangle" if self.read_unit is None else f"unit {self.read_unit}"
+        return (f"<micro_reader.Image {self.name!r} index={self.index} axes={self.axes!r} "
+                f"shape={self.shape} dtype={self.dtype} ({unit})>")
+
+
+#: the indices that say which image of a file an image is, outermost first.
+#: Arrays have only t c z y x (+ s); every other dimension is an index:
+#: illumination (light-sheet sides), phase, block (CZI), emission / excitation
+#: (spectral scans; the wavelength in the image's metadata), lifetime (FLIM
+#: bins) and sequence (a dimension the file does not name; open(...,
+#: rename={"sequence": "z"}) makes it an axis)
+INDEX_NAMES = ("scene", "tile", "rotation", "view", "illumination", "phase", "block",
+               "emission", "excitation", "lifetime", "sequence", "resolution")
+#: the indices that are not frames of the image but dimensions sampled at the
+#: same points (what other readers keep as array axes)
+SAMPLED_INDICES = ("illumination", "phase", "block", "emission", "excitation", "lifetime",
+                   "sequence")
+
+
+class File:
+    """An opened file: its images, the backend that reads them (``backend``:
+    "tiff", "czi", "nd2", "lif", "oir", "ims", "mrc", "raster") and the
+    format variant it is (``format``: e.g. "ome-tiff", "imagej", "lsm",
+    "nd2-legacy", "lof", "xlef", "png").  Every format opens as a File; a
+    backend's own class is an implementation detail."""
+
+    backend: str = "base"
+
+    def __init__(self, path: str, images: Sequence[Image]):
+        self.path = str(path)
+        self.images = list(images)
+        if not getattr(self, "format", None):
+            #: the format variant; the backend's name unless it reads several
+            self.format = self.backend
+        self._index_resolutions()
+
+    def _index_resolutions(self) -> None:
+        """``resolution`` is in every index when the file stores a pyramid of
+        any of its images (0 = full resolution, also for an image without
+        lower levels), and in none otherwise: an index holds the dimensions
+        the file has.  The same with resolutions="base" or "all"."""
+        members, seen = [], set()
+        for image in self.images:
+            for member in image.resolutions:
+                if id(member) not in seen:
+                    seen.add(id(member))
+                    members.append(member)
+        pyramidal = any(image.is_pyramidal for image in self.images)
+        for member in members:
+            index = {k: v for k, v in member.index.items() if k != "resolution"}
+            if pyramidal:
+                index["resolution"] = member.index.get("resolution", 0)
+            member.index = index
+
+    @property
+    def indices(self) -> list:
+        """Every image's index, in the order of ``images``."""
+        return [image.index for image in self.images]
+
+    @property
+    def index_range(self) -> dict:
+        """The widest range of each index in the file: how many distinct
+        values it takes, e.g. ``{"scene": 2, "tile": 5}``.  Not a total and
+        not a shape: with 3 tiles in scene 0 and 5 in scene 1, tile indices
+        run 0-2 and 0-4, so the range is 5 (not 8, and scene 0 has only 3).
+        Tile, rotation and view indices count from 0, so the values are
+        ``range(n)``; scene indices are the file's own (a CZI's may skip
+        numbers), so for scenes it is the number of distinct values."""
+        counts = {}
+        for name in INDEX_NAMES:
+            values = {image.index[name] for image in self.images if name in image.index}
+            if values:
+                counts[name] = len(values)
+        return counts
+
+    def select(self, **index) -> list:
+        """The images whose index matches, e.g. ``select(scene=1, view=0)``."""
+        unknown = set(index) - set(INDEX_NAMES)
+        if unknown:
+            raise KeyError(f"unknown indices {sorted(unknown)}; the indices are {INDEX_NAMES}")
+        return [image for image in self.images
+                if all(image.index.get(name) == value for name, value in index.items())]
+
+    def close(self) -> None:
+        for image in self.images:
+            image.close()
+
+    def __enter__(self) -> "File":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+    def __repr__(self) -> str:
+        return (f"<micro_reader.File {self.path!r} backend={self.backend!r} "
+                f"format={self.format!r} images={len(self.images)}>")