micro-reader 0.0.1__tar.gz → 0.0.2__tar.gz

{micro_reader-0.0.1 → micro_reader-0.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: micro-reader
-Version: 0.0.1
+Version: 0.0.2
 Summary: Low-level, memory-bounded pixel reading for microscopy formats: read exactly the rectangle you ask for.
 Project-URL: Homepage, https://github.com/bugraoezdemir/micro-reader
 Project-URL: Source, https://github.com/bugraoezdemir/micro-reader
@@ -677,7 +677,23 @@ You can also set it through the environment:
 export MICRO_READER_MEMORY_LIMIT=8GB
 ```
 
-For uncompressed data, only the requested rows and columns are read.
+The limit covers the blocks being decoded at once. Decoded blocks are also kept for reuse,
+so a block shared by neighbouring reads is not decoded twice. All open files share one
+cache limit, 256 MB by default:
+
+```python
+micro_reader.set_cache_limit("512MB")
+micro_reader.cache_limit()
+micro_reader.cached_bytes()     # what the caches hold now
+```
+
+or `export MICRO_READER_CACHE_LIMIT=512MB`. The process then uses at most the memory
+limit plus the cache limit for decoding, however many files are open. One exception: a
+single picture (PNG, JPEG, ... up to 256 MB) is kept decoded by its own open file.
+
+For uncompressed data, only the requested rows are read. When you ask for part of
+each row, the rest of the row is read along with it if that is faster than reading the
+rows one by one.
 
 Compressed data is stored in blocks such as TIFF strips or tiles, CZI subblocks, and ND2 frames. Small blocks are decoded as a whole. Blocks larger than 16 MB are read in parts where the codec allows it.
 

{micro_reader-0.0.1 → micro_reader-0.0.2}/README.md

@@ -625,7 +625,23 @@ You can also set it through the environment:
 export MICRO_READER_MEMORY_LIMIT=8GB
 ```
 
-For uncompressed data, only the requested rows and columns are read.
+The limit covers the blocks being decoded at once. Decoded blocks are also kept for reuse,
+so a block shared by neighbouring reads is not decoded twice. All open files share one
+cache limit, 256 MB by default:
+
+```python
+micro_reader.set_cache_limit("512MB")
+micro_reader.cache_limit()
+micro_reader.cached_bytes()     # what the caches hold now
+```
+
+or `export MICRO_READER_CACHE_LIMIT=512MB`. The process then uses at most the memory
+limit plus the cache limit for decoding, however many files are open. One exception: a
+single picture (PNG, JPEG, ... up to 256 MB) is kept decoded by its own open file.
+
+For uncompressed data, only the requested rows are read. When you ask for part of
+each row, the rest of the row is read along with it if that is faster than reading the
+rows one by one.
 
 Compressed data is stored in blocks such as TIFF strips or tiles, CZI subblocks, and ND2 frames. Small blocks are decoded as a whole. Blocks larger than 16 MB are read in parts where the codec allows it.
 

{micro_reader-0.0.1 → micro_reader-0.0.2}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "micro-reader"
-version = "0.0.1"
+version = "0.0.2"
 description = "Low-level, memory-bounded pixel reading for microscopy formats: read exactly the rectangle you ask for."
 readme = "README.md"
 requires-python = ">=3.11"

{micro_reader-0.0.1 → micro_reader-0.0.2}/src/micro_reader/__init__.py

@@ -8,6 +8,7 @@ from ._base import INDEX_NAMES, SAMPLED_INDICES, File, Image, ReadStats
 from ._metadata import Channel, Metadata, Quantity
 from ._errors import CorruptFileError, MicroReaderError, UnsupportedFormatError
 from ._memory import MemoryLimitError, memory_limit, set_memory_limit
+from ._io import cache_limit, cached_bytes, set_cache_limit
 from ._pool import decode_threads, set_decode_threads
 from ._registry import BACKENDS, open  # noqa: A004 - the public entry point
 from .backends.czi import DamagedFileWarning
@@ -18,6 +19,7 @@ __all__ = [
     "open", "File", "Image", "INDEX_NAMES", "SAMPLED_INDICES", "ReadStats", "BACKENDS",
     "Metadata", "Channel", "Quantity",
     "decode_threads", "set_decode_threads", "memory_limit", "set_memory_limit",
+    "cache_limit", "set_cache_limit", "cached_bytes",
     "MicroReaderError", "UnsupportedFormatError", "CorruptFileError", "MemoryLimitError", "DamagedFileWarning",
     "__version__",
 ]

micro_reader-0.0.2/src/micro_reader/_io.py

@@ -0,0 +1,159 @@
+"""Pieces every backend that reads segments on the decode pool needs."""
+from __future__ import annotations
+
+import builtins
+import threading
+from collections import OrderedDict
+
+import numpy as np
+
+from ._errors import CorruptFileError
+
+
+class ThreadHandles:
+    """One read-only file handle per thread for one file (seek + read on a
+    shared handle is not thread-safe), kept open until :meth:`close`."""
+
+    def __init__(self, path: str):
+        self.path = path
+        self._local = threading.local()
+        self._lock = threading.Lock()
+        self._all: list = []
+
+    def get(self):
+        fh = getattr(self._local, "fh", None)
+        if fh is None:
+            fh = self._local.fh = builtins.open(self.path, "rb", buffering=0)
+            with self._lock:
+                self._all.append(fh)
+        return fh
+
+    def read(self, offset: int, size: int) -> bytes:
+        fh = self.get()
+        fh.seek(offset)
+        data = fh.read(size)
+        if len(data) != size:
+            raise CorruptFileError(f"{self.path!r}: data at byte {offset} ends early "
+                                   "(truncated file?)")
+        return data
+
+    def close(self) -> None:
+        with self._lock:
+            handles, self._all = self._all, []
+        for fh in handles:
+            fh.close()
+
+
+#: what all decoded-block caches of a process may hold together, by default
+DEFAULT_CACHE_LIMIT = 256 << 20
+
+# One least-recently-used order over every cache of the process, so the number of
+# open files does not multiply the memory the caches hold: (cache id, key) ->
+# (cache, nbytes). Reentrant: a cache that is garbage-collected releases its bytes
+# from __del__, which may run while this thread holds the lock.
+_shared_lock = threading.RLock()
+_shared: OrderedDict = OrderedDict()
+_shared_bytes = 0
+_cache_limit = None
+_ids = iter(range(1 << 62))
+
+
+def cache_limit() -> int:
+    """What all decoded-block caches of this process may hold together, in bytes."""
+    global _cache_limit
+    if _cache_limit is None:
+        import os
+        from ._memory import parse_size
+        env = os.environ.get("MICRO_READER_CACHE_LIMIT")
+        _cache_limit = parse_size(env) if env else DEFAULT_CACHE_LIMIT
+    return _cache_limit
+
+
+def set_cache_limit(limit) -> None:
+    """Set what all decoded-block caches of this process may hold together (bytes or
+    e.g. "512MB"; 0 keeps nothing); None restores the default (the
+    ``MICRO_READER_CACHE_LIMIT`` environment variable, else 256 MB). Least recently
+    used blocks are dropped at once if the caches hold more."""
+    global _cache_limit
+    from ._memory import parse_size
+    with _shared_lock:
+        _cache_limit = None if limit is None else max(0, parse_size(limit))
+        _evict_shared(cache_limit())
+
+
+def cached_bytes() -> int:
+    """What the decoded-block caches of this process hold now, in bytes."""
+    with _shared_lock:
+        return _shared_bytes
+
+
+def _evict_shared(limit: int) -> None:
+    global _shared_bytes
+    while _shared_bytes > limit and _shared:
+        (_, key), (cache, nbytes) = _shared.popitem(last=False)
+        cache._drop(key)
+        _shared_bytes -= nbytes
+
+
+class DecodeCache:
+    """Decoded segments (tiles, strips, chunks), least recently used out.
+
+    Keeps a segment shared by neighbouring reads from being decoded twice.
+    Segments above a quarter of ``max_bytes`` are not kept (a whole plane from
+    a single-strip file would evict everything else). ``max_bytes`` caps this
+    cache (one file or scene); every cache of the process together stays within
+    :func:`cache_limit`, so many open files do not hold many times the memory."""
+
+    def __init__(self, max_bytes: int):
+        self.max_bytes = max_bytes
+        self._items: OrderedDict = OrderedDict()
+        self._bytes = 0
+        self._id = next(_ids)
+
+    def get(self, key):
+        with _shared_lock:
+            seg = self._items.get(key)
+            if seg is not None:
+                self._items.move_to_end(key)
+                _shared.move_to_end((self._id, key))
+            return seg
+
+    def put(self, key, seg: np.ndarray) -> None:
+        global _shared_bytes
+        if seg.nbytes > self.max_bytes // 4 or seg.nbytes > cache_limit():
+            return
+        seg.setflags(write=False)
+        with _shared_lock:
+            if key in self._items:
+                return
+            self._items[key] = seg
+            self._bytes += seg.nbytes
+            _shared[(self._id, key)] = (self, seg.nbytes)
+            _shared_bytes += seg.nbytes
+            while self._bytes > self.max_bytes:
+                old_key = next(iter(self._items))
+                _shared.pop((self._id, old_key))
+                _shared_bytes -= self._items[old_key].nbytes
+                self._drop(old_key)
+            _evict_shared(cache_limit())
+
+    def _drop(self, key) -> None:
+        """Forget *key* (the shared order is updated by the caller)."""
+        seg = self._items.pop(key, None)
+        if seg is not None:
+            self._bytes -= seg.nbytes
+
+    def clear(self) -> None:
+        global _shared_bytes
+        with _shared_lock:
+            for key, seg in self._items.items():
+                if _shared.pop((self._id, key), None) is not None:
+                    _shared_bytes -= seg.nbytes
+            self._items.clear()
+            self._bytes = 0
+
+    def __del__(self):
+        try:
+            self.clear()
+        except Exception:                           # noqa: BLE001 - interpreter exit
+            pass

{micro_reader-0.0.1 → micro_reader-0.0.2}/src/micro_reader/backends/czi.py

@@ -548,6 +548,20 @@ class _CziImage(Image):
             fh.seek(base + (y0 - sy0) * s.width * item)
             if fh.readinto(memoryview(buf).cast("B")) != buf.nbytes:
                 raise CorruptFileError(f"{self._czi.path!r}: subblock data ends early")
+        elif y1 - y0 > 1 and (s.width - width) * item <= SEPARATE_READ_BYTES:
+            # a few skipped bytes per row cost less than one more read per row: the span
+            # from the first row's columns to the last row's, in one read
+            rows, row = y1 - y0, s.width * self._samples
+            span = (rows - 1) * row + width * self._samples
+            flat = np.empty(span, self._file_dtype)
+            fh.seek(base + ((y0 - sy0) * s.width + (x0 - sx0)) * item)
+            if fh.readinto(memoryview(flat).cast("B")) != flat.nbytes:
+                raise CorruptFileError(f"{self._czi.path!r}: subblock data ends early")
+            buf = np.lib.stride_tricks.as_strided(
+                flat, shape=(rows, width * self._samples),
+                strides=(row * flat.itemsize, flat.itemsize))
+            self._place(buf, target, rows, width)
+            return span
         else:
             buf = np.empty((y1 - y0, width * self._samples), self._file_dtype)
             for i in range(y1 - y0):

{micro_reader-0.0.1 → micro_reader-0.0.2}/src/micro_reader/backends/mrc.py

@@ -31,6 +31,7 @@ from .._metadata import Channel, Metadata, quantity
 NAME = "mrc"
 HEADER_BYTES = 1024
 CHUNK_BYTES = 4 << 20        # full-width rows are read this many bytes at a time
+SEPARATE_READ_BYTES = 32 << 10  # bytes worth skipping rather than issuing one more read
 EXTENSIONS = {".mrc", ".mrcs", ".map", ".st", ".ali", ".rec"}
 
 #: MRC2014 mode -> numpy dtype.  Mode 0 is *signed* int8 in MRC2014; older
@@ -93,9 +94,13 @@ class _MrcImage(Image):
         # bottom of the image.  Rows are returned top-down, as Bio-Formats
         # (and so eubi-bridge, Fiji, OME-Zarr viewers) presents them: display
         # row y is file row ny-1-y.
+        read = out.size
         with builtins.open(self._path, "rb", buffering=0) as fh:
             if xs.stop - xs.start == nx:
                 self._read_full_rows(fh, zs, ys, out)
+            elif (ys.stop - ys.start > 1
+                  and (nx - (xs.stop - xs.start)) * item <= SEPARATE_READ_BYTES):
+                read = self._read_row_spans(fh, zs, ys, xs, out)
             else:
                 length = out.shape[2] * item
                 for i, z in enumerate(range(zs.start, zs.stop)):
@@ -107,7 +112,33 @@ class _MrcImage(Image):
                                 f"{self._path!r}: short read at section {z}")
         if not self._file_dtype.isnative:
             out = out.byteswap(inplace=True).view(self.dtype)
-        return out, out.size
+        return out, read
+
+    def _read_row_spans(self, fh, zs, ys, xs, out) -> int:
+        """Part of each row, when the columns skipped per row are few: they cost less
+        than one more read per row (as in LIF). Each chunk of rows is one read, from the
+        first row's columns to the last row's, reversed into place (bottom-up rows).
+        Returns the values read."""
+        nz, ny, nx = self.shape
+        item = self._file_dtype.itemsize
+        width = xs.stop - xs.start
+        per_chunk = max(1, CHUNK_BYTES // (nx * item))
+        flat = np.empty(min(per_chunk, ys.stop - ys.start) * nx, dtype=self._file_dtype)
+        first, stop = ny - ys.stop, ny - ys.start          # file rows, bottom-up
+        read = 0
+        for i, z in enumerate(range(zs.start, zs.stop)):
+            for f in range(first, stop, per_chunk):
+                n = min(per_chunk, stop - f)
+                span = (n - 1) * nx + width
+                fh.seek(self._offset + ((z * ny + f) * nx + xs.start) * item)
+                if fh.readinto(memoryview(flat[:span]).cast("B")) != span * item:
+                    raise CorruptFileError(f"{self._path!r}: short read at section {z}")
+                rows = np.lib.stride_tricks.as_strided(flat, shape=(n, width),
+                                                       strides=(nx * item, item))
+                top = ny - f - n - ys.start                   # display row of file row f+n-1
+                out[i, top:top + n] = rows[::-1]
+                read += span
+        return read
 
     def _read_full_rows(self, fh, zs, ys, out):
         """Full-width rows of a section are contiguous in the file: read them

{micro_reader-0.0.1 → micro_reader-0.0.2}/src/micro_reader/backends/nd2.py

@@ -36,6 +36,7 @@ from . import _nd2_structure as structure
 NAME = "nd2"
 DECODE_CACHE_BYTES = 64 << 20
 RAW_TASK_BYTES = 8 << 20          # uncompressed frames are read in row pieces of this size
+SEPARATE_READ_BYTES = 32 << 10    # bytes worth skipping rather than issuing one more read
 STREAM_PIECE_BYTES = 8 << 20      # rows of a streamed "lossless" frame decoded at a time
 TIMESTAMP_BYTES = 8               # each frame chunk starts with a float64 timestamp
 
@@ -168,7 +169,10 @@ class _Nd2Image(Image):
     def _raw_tasks(self, start, dst, cs, ys, xs, ss):
         f, a = self._f, self._f.attrs
         per_task = max(1, RAW_TASK_BYTES // a.width_bytes)
-        whole_rows = (xs.stop - xs.start) * 2 >= a.width     # else: only the needed bytes per row
+        # whole rows when the columns skipped per row are few: fewer bytes than another read
+        # call costs (as in LIF), or at most half the row; else only the needed bytes per row
+        skipped = a.width_bytes - (xs.stop - xs.start) * f.pixel_bytes
+        whole_rows = (xs.stop - xs.start) * 2 >= a.width or skipped <= SEPARATE_READ_BYTES
         for y0 in range(ys.start, ys.stop, per_task):
             y1 = min(ys.stop, y0 + per_task)
             part = dst[:, y0 - ys.start:y1 - ys.start]

{micro_reader-0.0.1 → micro_reader-0.0.2}/src/micro_reader/backends/tiff.py

@@ -54,6 +54,7 @@ NAME = "tiff"
 MAGIC = (b"II*\x00", b"MM\x00*", b"II+\x00", b"MM\x00+")   # classic and BigTIFF
 CHUNK_BYTES = 4 << 20            # buffered raw rows are read this many bytes at a time
 RAW_TASK_BYTES = 8 << 20         # uncompressed strip runs are split into pool tasks of this size
+SEPARATE_READ_BYTES = 32 << 10   # bytes worth skipping rather than issuing one more read
 DECODE_CACHE_BYTES = 64 << 20    # decoded segments kept per scene
 STREAM_PIECE_BYTES = 8 << 20     # rows of a streamed segment decoded at a time (reserved)
 #: image codecs that decode a rectangle alone (``_area_into``)
@@ -467,6 +468,25 @@ class _TiffImage(Image):
             fh.seek(offset + rows[0] * row_len * item)
             _readinto(fh, dst, self._path)
             return dst.size
+        width = cols[1] - cols[0]
+        if (cols != (0, stride) and rows[1] - rows[0] > 1
+                and (stride - width) * nc * item <= SEPARATE_READ_BYTES):
+            # a few skipped bytes per row cost less than one more read per row (as in
+            # LIF): read the span from the first row's columns to the last row's, in
+            # chunks, and keep the columns asked for
+            per_chunk = max(1, CHUNK_BYTES // (row_len * item))
+            flat = np.empty(min(per_chunk, rows[1] - rows[0]) * row_len, self._file_dtype)
+            read = 0
+            for a in range(rows[0], rows[1], per_chunk):
+                n = min(per_chunk, rows[1] - a)
+                span = (n - 1) * row_len + width * nc
+                fh.seek(offset + (a * stride + cols[0]) * nc * item)
+                _readinto(fh, flat[:span], self._path)
+                rows_view = np.lib.stride_tricks.as_strided(
+                    flat, shape=(n, width, nc), strides=(row_len * item, nc * item, item))
+                dst[a - rows[0]:a - rows[0] + n] = rows_view[..., cs]
+                read += span
+            return read
         if cols != (0, stride) and direct and dst[0].flags.c_contiguous:
             width = cols[1] - cols[0]
             for r in range(rows[0], rows[1]):

micro_reader-0.0.1/src/micro_reader/_io.py

@@ -1,83 +0,0 @@
-"""Pieces every backend that reads segments on the decode pool needs."""
-from __future__ import annotations
-
-import builtins
-import threading
-from collections import OrderedDict
-
-import numpy as np
-
-from ._errors import CorruptFileError
-
-
-class ThreadHandles:
-    """One read-only file handle per thread for one file (seek + read on a
-    shared handle is not thread-safe), kept open until :meth:`close`."""
-
-    def __init__(self, path: str):
-        self.path = path
-        self._local = threading.local()
-        self._lock = threading.Lock()
-        self._all: list = []
-
-    def get(self):
-        fh = getattr(self._local, "fh", None)
-        if fh is None:
-            fh = self._local.fh = builtins.open(self.path, "rb", buffering=0)
-            with self._lock:
-                self._all.append(fh)
-        return fh
-
-    def read(self, offset: int, size: int) -> bytes:
-        fh = self.get()
-        fh.seek(offset)
-        data = fh.read(size)
-        if len(data) != size:
-            raise CorruptFileError(f"{self.path!r}: data at byte {offset} ends early "
-                                   "(truncated file?)")
-        return data
-
-    def close(self) -> None:
-        with self._lock:
-            handles, self._all = self._all, []
-        for fh in handles:
-            fh.close()
-
-
-class DecodeCache:
-    """Decoded segments (tiles, strips, chunks), least recently used out.
-
-    Keeps a segment shared by neighbouring reads from being decoded twice.
-    Segments above a quarter of the budget are not kept (a whole plane from
-    a single-strip file would evict everything else)."""
-
-    def __init__(self, max_bytes: int):
-        self.max_bytes = max_bytes
-        self._items: OrderedDict = OrderedDict()
-        self._bytes = 0
-        self._lock = threading.Lock()
-
-    def get(self, key):
-        with self._lock:
-            seg = self._items.get(key)
-            if seg is not None:
-                self._items.move_to_end(key)
-            return seg
-
-    def put(self, key, seg: np.ndarray) -> None:
-        if seg.nbytes > self.max_bytes // 4:
-            return
-        seg.setflags(write=False)
-        with self._lock:
-            if key in self._items:
-                return
-            self._items[key] = seg
-            self._bytes += seg.nbytes
-            while self._bytes > self.max_bytes:
-                _, old = self._items.popitem(last=False)
-                self._bytes -= old.nbytes
-
-    def clear(self) -> None:
-        with self._lock:
-            self._items.clear()
-            self._bytes = 0