micro-reader 0.0.1__py3-none-any.whl

micro_reader/_registry.py

@@ -0,0 +1,104 @@
+"""Backend selection: content first, file extension only as a fallback."""
+from __future__ import annotations
+
+import builtins
+import os
+from typing import Optional
+
+from ._base import SAMPLED_INDICES, File
+from ._errors import UnsupportedFormatError
+from .backends import czi, ims, lif, mrc, nd2, oir, raster, tiff
+
+#: Tried in order: magic numbers before MRC, which may also match by
+#: extension alone.  Each backend module provides ``NAME``,
+#: ``sniff(head: bytes, path: str) -> bool`` and
+#: ``open(path, tiles="separate", resolutions="base") -> File``.
+BACKENDS = [tiff, czi, ims, nd2, oir, lif, raster, mrc]
+
+#: Bytes read from the start of the file for sniffing.
+HEAD_BYTES = 1024
+
+#: mosaic tiles as separate images (default) or composed, where the format
+#: defines how (CZI); see docs/AXES.md
+TILE_POLICIES = ("separate", "stitched")
+
+#: resolution levels: full resolution only (default), or every level as its
+#: own image indexed by "resolution" (image.resolutions gives a pyramid either way)
+RESOLUTION_POLICIES = ("base", "all")
+
+
+def open(path, backend: Optional[str] = None, *, tiles: str = "separate",  # noqa: A001
+         resolutions: str = "base", rename: Optional[dict] = None,
+         axes: Optional[str] = None, samples: str = "keep", squeeze: bool = False) -> File:
+    """Open *path* with the backend that recognises it (or the one named).
+
+    Each image of the file is one spatial frame with axes ``t c z y x`` (+ ``s``),
+    and an index saying which: scene, tile, rotation, view, and the
+    dimensions sampled at the same points (illumination, phase, block,
+    emission, excitation, lifetime, sequence).  *tiles*: ``"separate"`` gives
+    each mosaic tile as its own image; ``"stitched"`` composes a scene's
+    tiles, for formats that define how (CZI), and is an error for tiled files
+    of other formats.
+
+    *rename* makes an index an array axis, for files that do not say what a
+    dimension is: ``rename={"sequence": "z"}`` reads a plain multi-page TIFF
+    as one z-stack.  The target is ``t``, ``c`` or ``z``; a format that
+    cannot do it for that index raises.
+
+    *axes* (``"canonical"``, or letters such as ``"tczyx"``), *samples* and
+    *squeeze* present every image as ``Image.as_axes`` does.  Each image's
+    ``.source`` is then its native image."""
+    if tiles not in TILE_POLICIES:
+        raise ValueError(f"tiles={tiles!r}: one of {TILE_POLICIES}")
+    if resolutions not in RESOLUTION_POLICIES:
+        raise ValueError(f"resolutions={resolutions!r}: one of {RESOLUTION_POLICIES}")
+    rename = _check_rename(rename)
+    image = _open(os.fspath(path), backend, tiles, resolutions, rename)
+    left = sorted({name for index in image.indices for name in index if name in rename})
+    if left:
+        image.close()
+        raise UnsupportedFormatError(
+            f"rename: {path!r}: the {image.backend} backend cannot make {left} an axis")
+    if axes is not None or samples != "keep" or squeeze:
+        from ._axes import AxesImage
+        try:
+            image.images = [AxesImage(s, axes, samples=samples, squeeze=squeeze)
+                            for s in image.images]
+        except BaseException:
+            image.close()
+            raise
+    return image
+
+
+def _check_rename(rename: Optional[dict]) -> dict:
+    rename = {str(k).lower(): str(v).lower() for k, v in (rename or {}).items()}
+    for name, axis in rename.items():
+        if name not in SAMPLED_INDICES:
+            raise ValueError(f"rename: {name!r} is not an index that can become an axis "
+                             f"(those are {SAMPLED_INDICES})")
+        if axis not in ("t", "c", "z"):
+            raise ValueError(f"rename {name!r}: {axis!r} is not 't', 'c' or 'z'")
+    if len(set(rename.values())) != len(rename):
+        raise ValueError(f"rename {rename}: two indices onto one axis")
+    return rename
+
+
+def _open(path: str, backend: Optional[str], tiles: str, resolutions: str,
+          rename: dict) -> File:
+    def call(module):
+        if rename and getattr(module, "RENAMES", False):
+            return module.open(path, tiles, resolutions, rename=rename)
+        return module.open(path, tiles, resolutions)
+
+    if backend is not None:
+        for module in BACKENDS:
+            if module.NAME == backend:
+                return call(module)
+        raise UnsupportedFormatError(
+            f"unknown backend {backend!r}; available: {[m.NAME for m in BACKENDS]}")
+    with builtins.open(path, "rb") as fh:
+        head = fh.read(HEAD_BYTES)
+    for module in BACKENDS:
+        if module.sniff(head, path):
+            return call(module)
+    raise UnsupportedFormatError(f"no micro-reader backend recognises {path!r}")

micro_reader/_stream.py

@@ -0,0 +1,317 @@
+"""Streaming decode of compressed blocks too large to decode whole
+(``_memory``; docs/PLAN.md, "memory-bounded reads of huge compressed blocks").
+
+A block is decoded from its start, keeping only the bytes a read asks for;
+the compressed bytes are read from the file in small pieces.  Each decoder
+keeps its place, and reads are handed the decoder of their block that is
+closest before their start, so regions read top to bottom (as eubi-bridge's
+writer reads) continue where the last one stopped: reading a whole plane
+stays linear.  Several reads of one block at once each get a decoder of
+their own.
+
+Checkpoints (deflate / zlib): the decoders of a block share an index of the
+inflater's state every ``every`` decoded bytes (about 1/128 of the block, at
+least 1 MB; ~43 KB each), so a read before a decoder's place -- a viewer
+jumping around -- resumes from the nearest checkpoint instead of starting
+the block again.  Indexes of the ``MAX_INDEXES`` most recent blocks are kept.
+
+Native: ``micro_reader_codecs.StreamDecoder`` (zlib / deflate, LZW, zstd,
+PackBits, none).  Reference, in Python: zlib / deflate (stdlib), LZMA
+(stdlib), zstd (Python 3.14 or ``zstandard``), PackBits, none -- LZMA
+always, the others when the native package is missing.
+"""
+from __future__ import annotations
+
+import threading
+import zlib
+from collections import OrderedDict
+
+from . import _codecs
+from ._errors import CorruptFileError
+
+#: codecs that can be streamed
+STREAMABLE = ("zlib", "deflate", "lzw", "zstd", "packbits", "lzma", "none", "png")
+NATIVE = ("zlib", "deflate", "lzw", "zstd", "packbits", "none", "png")
+#: decoders kept for later reads (each holds a file handle and its window)
+MAX_IDLE = 32
+#: blocks whose checkpoint indexes are kept; checkpoints per block, spacing
+MAX_INDEXES = 16
+MAX_CHECKPOINTS = 128
+MIN_CHECKPOINT_EVERY = 1 << 20
+CHECKPOINTED = ("zlib", "deflate")
+_PIECE = 1 << 16
+
+
+def streamable(codec: str) -> bool:
+    if codec not in STREAMABLE:
+        return False
+    if codec in NATIVE and _native_stream() is not None:
+        return True
+    return codec in ("zlib", "deflate", "lzma", "packbits", "none") or \
+        (codec == "zstd" and _reference_zstd_stream() is not None)
+
+
+def _native_stream():
+    native = _codecs._native
+    return getattr(native, "StreamDecoder", None) if native is not None else None
+
+
+def _reference_zstd_stream():
+    try:
+        from compression import zstd                  # Python >= 3.14
+        return lambda: zstd.ZstdDecompressor()
+    except ImportError:
+        pass
+    try:
+        import zstandard
+        return lambda: zstandard.ZstdDecompressor().decompressobj()
+    except ImportError:
+        return None
+
+
+class _PackBits:
+    """PackBits as a decompressor object (``decompress(data)``)."""
+
+    def __init__(self):
+        self.pending = b""
+
+    def decompress(self, data: bytes) -> bytes:
+        data, out, i = self.pending + data, bytearray(), 0
+        while i < len(data):
+            n = data[i] - 256 if data[i] > 127 else data[i]
+            if n >= 0:
+                if i + 1 + n + 1 > len(data):
+                    break
+                out += data[i + 1:i + 2 + n]
+                i += n + 2
+            elif n != -128:
+                if i + 1 >= len(data):
+                    break
+                out += bytes([data[i + 1]]) * (1 - n)
+                i += 2
+            else:
+                i += 1
+        self.pending = data[i:]
+        return bytes(out)
+
+
+class _ReferenceIndex:
+    """Checkpoints of one zlib / deflate block for the reference decoders:
+    (decoded position, compressed bytes consumed, decompressor copy)."""
+
+    def __init__(self, every: int, max: int = MAX_CHECKPOINTS):
+        self.every, self.max, self.points = every, max, []
+        self._lock = threading.Lock()
+
+    def best(self, start: int):
+        with self._lock:
+            before = [p for p in self.points if p[0] <= start]
+            return max(before, key=lambda p: p[0]) if before else None
+
+    def record(self, position: int, consumed: int, decompressor) -> None:
+        with self._lock:
+            if len(self.points) >= self.max or \
+                    any(abs(p[0] - position) < self.every for p in self.points):
+                return
+            self.points.append((position, consumed, decompressor.copy()))
+
+    def __len__(self):
+        return len(self.points)
+
+
+class _ReferenceDecoder:
+    """The native decoder's interface, decoded with Python's decompressor objects."""
+
+    def __init__(self, path: str, offset: int, length: int, codec: str, index=None):
+        self.path, self.offset, self.length, self.codec = path, offset, length, codec
+        self.position, self.restarts, self.resumes = 0, 0, 0
+        self._index = index if codec in CHECKPOINTED else None
+        self._fh = None
+        self._lock = threading.Lock()
+
+    def _resume(self, point) -> None:
+        position, consumed, decompressor = point
+        if self._fh is not None:
+            self._fh.close()
+        self._fh = open(self.path, "rb")                   # noqa: SIM115 - kept open
+        self._fh.seek(self.offset + consumed)
+        self._left = self.length - consumed
+        self._pending = b""
+        self.position = position
+        self._decompressor = decompressor.copy()
+        self.resumes += 1
+
+    def _start(self):
+        import lzma
+        if self._fh is not None:
+            self.restarts += 1
+            self._fh.close()
+        self._fh = open(self.path, "rb")                   # noqa: SIM115 - kept open
+        self._fh.seek(self.offset)
+        self._left = self.length
+        self._pending = b""
+        self.position = 0
+        self._decompressor = {
+            "zlib": lambda: zlib.decompressobj(),
+            "deflate": lambda: zlib.decompressobj(-15),
+            "lzma": lambda: lzma.LZMADecompressor(),
+            "zstd": _reference_zstd_stream(),
+            "packbits": _PackBits,
+            "none": lambda: None,
+        }[self.codec]()
+
+    def _more(self) -> bytes:
+        """The next decoded bytes; b"" at the end of the block."""
+        while True:
+            if not self._left:
+                return b""
+            data = self._fh.read(min(_PIECE, self._left))
+            self._left -= len(data)
+            if not data:
+                self._left = 0
+                return b""
+            if self._decompressor is None:
+                return data
+            try:
+                out = self._decompressor.decompress(data)
+            except Exception as exc:                    # noqa: BLE001 - codec specific
+                raise CorruptFileError(f"{self.path!r}: corrupt {self.codec} data: {exc}") from exc
+            if out:
+                return out
+
+    def read_into(self, start: int, out) -> int:
+        view = memoryview(out).cast("B")
+        with self._lock:
+            behind = self._fh is None or start < self.position
+            point = self._index.best(start) if self._index is not None else None
+            if point is not None and (behind or point[0] > self.position):
+                self._resume(point)
+            elif behind:
+                self._start()
+            written = 0
+            while written < len(view):
+                if not self._pending:
+                    if self._index is not None:          # all decoded so far is delivered
+                        self._index.record(self.position, self.length - self._left,
+                                           self._decompressor)
+                    self._pending = self._more()
+                    if not self._pending:
+                        break
+                skip = start + written - self.position
+                if skip >= len(self._pending):             # all before the start
+                    self.position += len(self._pending)
+                    self._pending = b""
+                    continue
+                if skip > 0:
+                    self.position += skip
+                    self._pending = self._pending[skip:]
+                n = min(len(view) - written, len(self._pending))
+                view[written:written + n] = self._pending[:n]
+                self._pending = self._pending[n:]
+                self.position += n
+                written += n
+            return written
+
+    def close(self) -> None:
+        with self._lock:
+            if self._fh is not None:
+                self._fh.close()
+                self._fh = None
+
+    def __del__(self):
+        if getattr(self, "_fh", None) is not None:
+            self._fh.close()
+
+
+_lock = threading.Lock()
+#: (path, offset, codec) -> decoders not in use; least recently used first
+_idle: "OrderedDict[tuple, list]" = OrderedDict()
+#: (path, offset, codec) -> the block's checkpoint index; least recently used first
+_indexes: "OrderedDict[tuple, object]" = OrderedDict()
+
+
+def _index(key: tuple, size) -> object:
+    """The checkpoint index of block *key* (zlib / deflate), created on first use."""
+    if key[2] not in CHECKPOINTED:
+        return None
+    with _lock:
+        index = _indexes.get(key)
+        if index is None:
+            every = max(MIN_CHECKPOINT_EVERY, int(size or 0) // MAX_CHECKPOINTS)
+            native = _codecs._native
+            index = (native.BlockIndex(every, MAX_CHECKPOINTS)
+                     if native is not None and hasattr(native, "BlockIndex")
+                     else _ReferenceIndex(every))
+            _indexes[key] = index
+            while len(_indexes) > MAX_INDEXES:
+                _indexes.popitem(last=False)
+        else:
+            _indexes.move_to_end(key)
+        return index
+
+
+def _new(path: str, offset: int, length: int, codec: str, index=None):
+    native = _native_stream()
+    if native is not None and codec in NATIVE:
+        if index is not None and not isinstance(index, _ReferenceIndex):
+            return native(path, offset, length, codec, index)
+        return native(path, offset, length, codec)
+    return _ReferenceDecoder(path, offset, length, codec,
+                             index if isinstance(index, _ReferenceIndex) else None)
+
+
+def _acquire(key: tuple, length: int, start: int, size=None):
+    with _lock:
+        decoders = _idle.get(key, [])
+        before = [d for d in decoders if d.position <= start]
+        if before:
+            decoder = max(before, key=lambda d: d.position)
+            decoders.remove(decoder)
+            if not decoders:
+                _idle.pop(key, None)
+            return decoder
+        if decoders:                     # all past *start*: one resumes from a checkpoint
+            decoder = decoders.pop(0)
+            if not decoders:
+                _idle.pop(key, None)
+            return decoder
+    return _new(key[0], key[1], length, key[2], _index(key, size))
+
+
+def _release(key: tuple, decoder) -> None:
+    with _lock:
+        _idle.setdefault(key, []).append(decoder)
+        _idle.move_to_end(key)
+        total = sum(len(v) for v in _idle.values())
+        while total > MAX_IDLE:
+            oldest = next(iter(_idle))
+            _idle[oldest].pop(0)
+            if not _idle[oldest]:
+                del _idle[oldest]
+            total -= 1
+
+
+def read(path: str, offset: int, length: int, codec: str, start: int, out,
+         size: int = None) -> int:
+    """Decoded bytes *start*... of the block (*length* compressed bytes at
+    *offset* of *path*, *codec*; *size* decoded bytes, if known) into *out*;
+    returns the bytes written (fewer at the end of the block)."""
+    key = (str(path), int(offset), codec)
+    decoder = _acquire(key, int(length), int(start), size)
+    try:
+        try:
+            return decoder.read_into(int(start), out)
+        except ValueError as exc:
+            raise CorruptFileError(f"{path!r}: corrupt {codec} data: {exc}") from exc
+    finally:
+        _release(key, decoder)
+
+
+def forget(path: str) -> None:
+    """Close and drop the decoders of *path* (its file was closed)."""
+    with _lock:
+        dropped = [d for key in [k for k in _idle if k[0] == str(path)] for d in _idle.pop(key)]
+        for key in [k for k in _indexes if k[0] == str(path)]:
+            del _indexes[key]
+    for decoder in dropped:
+        getattr(decoder, "close", lambda: None)()

micro_reader/backends/__init__.py

@@ -0,0 +1 @@
+"""Format backends.  Each module provides NAME, sniff(head, path) and open(path)."""

micro_reader/backends/_czi_metadata.py

@@ -0,0 +1,188 @@
+"""Metadata of CZI images (docs/METADATA.md), from the file's XML metadata
+segment (``ZISRAWMETADATA``) and the first subblock's own metadata.
+
+- Pixel sizes: ``Metadata/Scaling/Items/Distance`` X / Y / Z, stored in
+  metres; given in the file's display unit (``DefaultUnitFormat``, normally
+  µm) when OME-Zarr has a name for it, else micrometres.
+- Time step: ``Information/Image/Dimensions/T/Positions/Interval/Increment``
+  (seconds).
+- Channels: ``Information/Image/Dimensions/Channels/Channel`` (Name, Color,
+  Excitation / EmissionWavelength in nm), colours missing there from
+  ``DisplaySetting/Channels``.
+- Stage position: of the centre of the image's first pixel (docs/METADATA.md),
+  from ``StageXPosition`` / ``StageYPosition`` (µm) of the image's first
+  subblock, which Zeiss records for the centre of that field of view
+  (Bio-Formats reports it as is: the centre); files without it: the
+  scene's ``Positions/Position`` in the XML, taken as the image's centre.
+"""
+from __future__ import annotations
+
+import struct
+from dataclasses import replace
+import xml.etree.ElementTree as ET
+
+from .._metadata import Channel, Metadata, Quantity, color_hex, quantity, unit_name
+
+
+def read_xml(fh):
+    """The root of the XML metadata segment of the CZI open as *fh*, or
+    None (no segment, unreadable XML)."""
+    fh.seek(32)
+    head = fh.read(80)
+    metadata_position = struct.unpack_from("<iiii16s16siqqiq", head)[8]
+    if not metadata_position:
+        return None
+    fh.seek(metadata_position)
+    seg = fh.read(32 + 8)
+    if seg[:14] != b"ZISRAWMETADATA":
+        return None
+    size, _attachment = struct.unpack_from("<ii", seg, 32)
+    fh.seek(metadata_position + 32 + 256)
+    try:
+        return ET.fromstring(fh.read(size))
+    except ET.ParseError:
+        return None
+
+
+def subblock_tags(fh, position: int) -> dict:
+    """The ``Tags`` of the subblock segment at *position*: {tag: text}."""
+    fh.seek(position)
+    head = fh.read(32 + 48)
+    if head[:14] != b"ZISRAWSUBBLOCK":
+        return {}
+    metadata_size, = struct.unpack_from("<i", head, 32)
+    ndims, = struct.unpack_from("<i", head, 32 + 44)
+    if metadata_size <= 0:
+        return {}
+    fh.seek(position + 32 + 16 + max(240, 32 + 20 * ndims))
+    try:
+        root = ET.fromstring(fh.read(metadata_size).rstrip(b"\x00"))
+    except ET.ParseError:
+        return {}
+    tags = root.find("Tags")
+    return {} if tags is None else {e.tag: (e.text or "").strip() for e in tags}
+
+
+def pixel_sizes(root) -> dict:
+    sizes = {}
+    if root is None:
+        return sizes
+    for d in root.findall("Metadata/Scaling/Items/Distance"):
+        axis = (d.get("Id") or "").lower()
+        if axis not in ("x", "y", "z"):
+            continue
+        q = quantity(d.findtext("Value"), "meter")
+        if q is None:
+            continue
+        shown = unit_name(d.findtext("DefaultUnitFormat"))
+        # the display unit when it is a length OME-Zarr names (and not
+        # larger than a millimetre: ``quantity``), else µm
+        try:
+            shown_q = q.to(shown) if shown else q
+        except KeyError:
+            shown_q = q
+        sizes[axis] = quantity(shown_q.value, shown_q.unit)
+    increment = root.findtext("Metadata/Information/Image/Dimensions/T/Positions/Interval/Increment")
+    q = quantity(increment, "second")
+    if q:
+        sizes["t"] = q
+    return sizes
+
+
+def channels(root, c_start: int, n: int) -> tuple:
+    """Channels with indices c_start .. c_start + n - 1 of the file."""
+    found = []
+    display = []
+    if root is not None:
+        found = root.findall("Metadata/Information/Image/Dimensions/Channels/Channel")
+        display = root.findall("Metadata/DisplaySetting/Channels/Channel")
+    display_color = {}
+    for k, d in enumerate(display):
+        color = color_hex(d.findtext("Color"))
+        for name in (d.get("Name"), d.get("Id"), k):
+            display_color.setdefault(name, color)
+    out = []
+    for k in range(c_start, c_start + max(1, n)):
+        if k >= len(found):
+            out.append(Channel())
+            continue
+        c = found[k]
+        name = c.get("Name") or None
+        color = color_hex(c.findtext("Color"))
+        if color is None:
+            color = display_color.get(name) or display_color.get(c.get("Id"))
+            if color is None and len(display) == len(found):
+                color = display_color.get(k)
+        out.append(Channel(
+            name=name, color=color,
+            emission_wavelength=quantity(c.findtext("EmissionWavelength"), "nanometer"),
+            excitation_wavelength=quantity(c.findtext("ExcitationWavelength"), "nanometer")))
+    return tuple(out)
+
+
+def scene_position(root, scene: int) -> dict:
+    """The scene's stage position in the XML (µm): ``Positions/Position``."""
+    if root is None:
+        return {}
+    for e in root.findall("Metadata/Information/Image/Dimensions/S/Scenes/Scene"):
+        if e.get("Index") == str(scene):
+            p = e.find("Positions/Position")
+            if p is not None:
+                out = {a: quantity(p.get(a.upper()), "micrometer", positive=False)
+                       for a in ("x", "y")}
+                return {a: q for a, q in out.items() if q}
+    return {}
+
+
+def position_from_tags(tags: dict) -> dict:
+    out = {}
+    for axis in ("x", "y"):
+        q = quantity(tags.get(f"Stage{axis.upper()}Position"), "micrometer", positive=False)
+        if q:
+            out[axis] = q
+    return out
+
+
+def first_pixel_position(image, centre: dict, s, sizes: dict) -> dict:
+    """Stage position of the centre of *image*'s first pixel, from *centre*:
+    the stage position of the centre of subblock *s* (Zeiss records where
+    the field of view's centre was).  Pixel coordinates are one frame for
+    the whole file, so the offset between the two frames found at *s* holds
+    for the image's corner too."""
+    out = {}
+    origin = dict(zip("xy", image.origin))
+    extent = {"x": s.width, "y": s.height}
+    start = {"x": s.x, "y": s.y}
+    for axis, q in centre.items():
+        size = sizes.get(axis)
+        if size is None:
+            continue
+        step = size.to(q.unit).value
+        offset = q.value - (start[axis] + extent[axis] / 2) * step
+        out[axis] = Quantity(offset + (origin[axis] + 0.5) * step, q.unit)
+    return out
+
+
+def czi_metadata(image, fh, root) -> Metadata:
+    """*image*'s Metadata; *fh* an open handle of its file, *root* the
+    file's XML metadata (or None)."""
+    axes = image.axes
+    n_c = image.shape[axes.index("c")] if "c" in axes else 1
+    sizes = pixel_sizes(root)
+    # the first subblock drawn of the first plane
+    first = image._planes[min(image._planes)].subblocks[0] if image._planes else None
+    position = {}
+    if first is not None and first.position >= 0:
+        centre = position_from_tags(subblock_tags(fh, first.position))
+        position = first_pixel_position(image, centre, first, sizes)
+    if not position and first is not None:
+        # no stage position per subblock: the scene's, taken as the centre of
+        # the whole scene (all its level-0 tiles; as Bio-Formats places it)
+        tiles = [e for e in image._czi._level0 if e.scene == first.scene]
+        x0, y0 = min(e.x for e in tiles), min(e.y for e in tiles)
+        x1 = max(e.x + e.width for e in tiles)
+        y1 = max(e.y + e.height for e in tiles)
+        whole = replace(first, x=x0, y=y0, width=x1 - x0, height=y1 - y0)
+        position = first_pixel_position(image, scene_position(root, first.scene), whole, sizes)
+    return Metadata(pixel_sizes=sizes, position=position,
+                    channels=channels(root, image._czi._dim_starts.get("C", 0), n_c))