micro-reader 0.0.1__py3-none-any.whl

micro_reader/_jpeg_restarts.py

@@ -0,0 +1,174 @@
+"""Rows of a JPEG too large to decode whole, decoded alone through its
+restart markers (docs/PLAN.md, "memory-bounded reads of huge compressed
+blocks", step 4).
+
+A sequential JPEG with a restart interval (DRI) resets its entropy coder
+every *Ri* MCUs and marks the place (RST0..RST7).  The intervals covering
+some MCU rows, put behind the JPEG's own header with the frame height
+changed, make a JPEG of those rows alone: any decoder decodes it, and only
+it is held in memory.  Pixels are the same as a whole decode: blocks decode
+alone, and one MCU row more above and below is decoded and dropped where
+chroma is subsampled (upsampling looks at the neighbouring rows).
+
+Cut only:
+- baseline / extended sequential JPEG (SOF0, SOF1), one scan of all
+  components;
+- with intervals that start on MCU rows often enough: every
+  lcm(Ri, MCUs per row) / MCUs per row MCU rows (``group``).
+
+Progressive JPEG (its refinement scans hold the whole image), lossless and
+JPEG without restart markers return None from ``RestartIndex.build`` and are
+decoded whole, within the memory limit.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+import struct
+
+import numpy as np
+
+#: RST0..RST7 and EOI in entropy-coded data (0xFF 0x00 is a stuffed 0xFF)
+_MARKERS = re.compile(rb"\xff[\xd0-\xd7\xd9]")
+
+
+class RestartIndex:
+    """Where the restart intervals of one sequential JPEG start and end, in
+    bytes of its codestream; ``build`` it from the codestream."""
+
+    def __init__(self, header: bytes, sof: int, height: int, width: int,
+                 mcu_rows: int, mcu_height: int, group: int, starts, ends, subsampled: bool,
+                 intervals_per_group: int):
+        self.header = header                    # SOI .. the scan header, kept
+        self._sof = sof                         # offset of the frame height in *header*
+        self.height, self.width = height, width
+        self.mcu_rows, self.mcu_height = mcu_rows, mcu_height
+        #: MCU rows between interval starts that fall on an MCU row
+        self.group = group
+        self._per_group = intervals_per_group
+        self._starts, self._ends = starts, ends
+        self._overlap = 1 if subsampled else 0  # groups decoded above / below and dropped
+
+    @classmethod
+    def build(cls, data) -> "RestartIndex | None":
+        """The index of the JPEG codestream *data*, or None when it cannot
+        be cut (see the module docstring)."""
+        data = memoryview(data).cast("B")
+        if bytes(data[:2]) != b"\xff\xd8":
+            return None
+        pos, sof, frame, interval, scan = 2, None, None, 0, None
+        while pos + 4 <= len(data):
+            if data[pos] != 0xFF:
+                return None
+            marker = data[pos + 1]
+            if marker == 0xFF:                  # fill byte
+                pos += 1
+                continue
+            length = struct.unpack(">H", data[pos + 2:pos + 4])[0]
+            body = bytes(data[pos + 4:pos + 2 + length])
+            if marker in (0xC0, 0xC1):
+                if frame is not None:
+                    return None
+                sof = pos + 5
+                _bits, height, width, n = struct.unpack(">BHHB", body[:6])
+                frame = (height, width, [(body[7 + 3 * k] >> 4, body[7 + 3 * k] & 15)
+                                         for k in range(n)])
+            elif 0xC2 <= marker <= 0xCF and marker not in (0xC4, 0xC8, 0xCC):
+                return None                     # progressive, lossless, arithmetic
+            elif marker == 0xDD:
+                interval = struct.unpack(">H", body[:2])[0]
+            elif marker == 0xDA:
+                if frame is None or body[0] != len(frame[2]):
+                    return None                 # a scan of only some components
+                scan = pos + 2 + length
+                break
+            pos += 2 + length
+        if scan is None or interval == 0 or frame[0] == 0:
+            return None
+        height, width, sampling = frame
+        if len(sampling) == 1:                  # one component: 8 x 8 MCUs
+            mcu_w = mcu_h = 8
+        else:
+            mcu_w, mcu_h = 8 * max(h for h, _ in sampling), 8 * max(v for _, v in sampling)
+        across = -(-width // mcu_w)
+        mcu_rows = -(-height // mcu_h)
+        group = math.lcm(interval, across) // across
+        markers = [(m.start(), data[m.start() + 1]) for m in _MARKERS.finditer(data, scan)]
+        end = next((p for p, kind in markers if kind == 0xD9), len(data))
+        rst = [p for p, kind in markers if kind != 0xD9 and p < end]
+        starts = np.array([scan] + [p + 2 for p in rst], np.int64)
+        ends = np.array(rst + [end], np.int64)
+        if len(starts) != -(-across * mcu_rows // interval):
+            return None                         # markers missing or extra: decode whole
+        subsampled = len({s for s in sampling}) > 1
+        return cls(bytes(data[:scan]), sof, height, width, mcu_rows, mcu_h, group,
+                   starts, ends, subsampled, group * across // interval)
+
+    def group_bytes(self, row_bytes: int) -> int:
+        """Decoded bytes of one group of MCU rows, *row_bytes* per image row."""
+        return self.group * self.mcu_height * row_bytes
+
+    def smallest_band(self, row_bytes: int) -> int:
+        """Memory of the smallest band ``read_rows`` decodes: one group, the
+        overlap above and below, and the decoder's own planes."""
+        return 2 * (1 + 2 * self._overlap) * self.group_bytes(row_bytes)
+
+
+    def bands(self, y0: int, y1: int, groups: int):
+        """(g0, g1): the groups of MCU rows covering image rows *y0*..*y1*,
+        *groups* at a time (``codestream`` adds the overlap)."""
+        rows_per_group = self.group * self.mcu_height
+        g0, g1 = y0 // rows_per_group, -(-y1 // rows_per_group)
+        for g in range(g0, g1, groups):
+            yield g, min(g1, g + groups)
+
+    def codestream(self, read, g0: int, g1: int):
+        """The JPEG of groups *g0*..*g1* (plus the overlap) and the image row
+        it starts at.  *read(start, stop)*: bytes start..stop of the
+        original codestream."""
+        groups = -(-self.mcu_rows // self.group)
+        g0, g1 = max(0, g0 - self._overlap), min(groups, g1 + self._overlap)
+        j0, j1 = g0 * self._per_group, min(len(self._starts), g1 * self._per_group)
+        first = g0 * self.group * self.mcu_height
+        stop = min(self.height, g1 * self.group * self.mcu_height)
+        header = bytearray(self.header)
+        header[self._sof:self._sof + 2] = struct.pack(">H", stop - first)
+        raw = read(int(self._starts[j0]), int(self._ends[j1 - 1]))
+        base = int(self._starts[j0])
+        parts = [bytes(header)]
+        for j in range(j0, j1):
+            parts.append(raw[self._starts[j] - base:self._ends[j] - base])
+            if j < j1 - 1:
+                parts.append(bytes((0xFF, 0xD0 + (j - j0) % 8)))
+        parts.append(b"\xff\xd9")
+        return b"".join(parts), first, stop
+
+
+def usable(index: "RestartIndex | None", row_bytes: int, fits) -> bool:
+    """Whether *index* cuts its JPEG in bands that *fits(nbytes)* allows:
+    intervals that seldom start on an MCU row make every band large."""
+    return index is not None and fits(index.smallest_band(row_bytes))
+
+
+def read_rows(index: RestartIndex, read, y0: int, y1: int, decode, shape_tail, dtype,
+              piece_bytes: int, reserve):
+    """Image rows *y0*..*y1* of a cut JPEG, in bands: yields (a, b, rows),
+    rows *a*..*b* of the image as (b - a, width, *shape_tail*).  *decode(data,
+    out)* decodes a JPEG into *out*; *reserve(nbytes)* holds the memory
+    budget while a band is decoded (the band and the decoder's own planes)."""
+    row_bytes = index.width * int(np.prod(shape_tail, dtype=np.int64)) * np.dtype(dtype).itemsize
+    groups = max(1, piece_bytes // max(1, index.group_bytes(row_bytes)))
+    for g0, g1 in index.bands(y0, y1, groups):
+        data, first, stop = index.codestream(read, g0, g1)
+        band = np.empty((stop - first, index.width) + tuple(shape_tail), dtype)
+        with reserve(2 * band.nbytes):                          # the band and the decoder's
+            got = decode(data, band)
+            if (got[1], got[0]) != band.shape[:2]:
+                raise ValueError(f"a cut JPEG decodes to {got[1]} x {got[0]}, "
+                                 f"expected {band.shape[0]} x {band.shape[1]}")
+            rows_per_group = index.group * index.mcu_height       # the band's own rows,
+            a = max(y0, g0 * rows_per_group)                      # not the overlap
+            b = min(y1, g1 * rows_per_group, index.height)
+            if a < b:
+                yield a, b, band[a - first:b - first]

micro_reader/_memory.py

@@ -0,0 +1,161 @@
+"""The memory budget for decoding (docs/PLAN.md, "memory-bounded reads of huge
+compressed blocks").
+
+Compressed data is decoded per block (a TIFF strip or tile, a CZI subblock,
+an ND2 "lossless" frame, a whole PNG ...).
+
+- A block too large for the decoded-block caches (``STREAM_THRESHOLD``) is
+  streamed where its codec allows it: decoded from its start, keeping only
+  the rows a read needs.  Decoding it whole would buy nothing -- it cannot
+  be kept for the next read -- and costs the whole block for every read.
+- Every other block is decoded whole, and reserves its size (``reserve``)
+  while it is being decoded: the blocks in flight never exceed
+  ``memory_limit()`` together, whichever threads decode them (a caller's
+  own threads as well as the decode pool); a decode that does not fit
+  waits until others finish.  A single block above the limit raises
+  ``MemoryLimitError``.
+
+Size: ``set_memory_limit(n)`` (bytes, or a string such as ``"8GB"``), else
+the ``MICRO_READER_MEMORY_LIMIT`` environment variable, else a quarter of
+the computer's memory.  The limit is per process: each worker process of a
+caller such as eubi-bridge sets its own.
+"""
+from __future__ import annotations
+
+import os
+import re
+import threading
+from typing import Optional, Union
+
+from ._errors import MicroReaderError
+
+_lock = threading.Lock()
+_limit: Optional[int] = None
+#: when the computer's memory cannot be found
+FALLBACK_LIMIT = 4 << 30
+#: never below this: a budget smaller than one camera frame helps nobody
+MINIMUM_LIMIT = 64 << 20
+_UNITS = {"": 1, "b": 1, "k": 1 << 10, "kb": 1 << 10, "kib": 1 << 10, "m": 1 << 20,
+          "mb": 1 << 20, "mib": 1 << 20, "g": 1 << 30, "gb": 1 << 30, "gib": 1 << 30,
+          "t": 1 << 40, "tb": 1 << 40, "tib": 1 << 40}
+
+
+class MemoryLimitError(MicroReaderError):
+    """A block that must be decoded whole does not fit the memory budget."""
+
+
+def parse_size(value: Union[int, float, str]) -> int:
+    """Bytes from 4294967296, "4GB", "512 MB", "1.5g" (binary units)."""
+    if isinstance(value, (int, float)):
+        return int(value)
+    match = re.fullmatch(r"\s*([0-9]*\.?[0-9]+)\s*([a-zA-Z]*)\s*", str(value))
+    if not match or match.group(2).lower() not in _UNITS:
+        raise ValueError(f"memory size {value!r}: a number of bytes or e.g. '8GB'")
+    return int(float(match.group(1)) * _UNITS[match.group(2).lower()])
+
+
+def physical_memory() -> Optional[int]:
+    """The computer's memory in bytes, or None if it cannot be found."""
+    try:
+        if os.name == "nt":
+            import ctypes
+
+            class _Status(ctypes.Structure):
+                _fields_ = [("length", ctypes.c_ulong), ("load", ctypes.c_ulong),
+                            ("total", ctypes.c_ulonglong), ("free", ctypes.c_ulonglong),
+                            ("page_total", ctypes.c_ulonglong), ("page_free", ctypes.c_ulonglong),
+                            ("virtual_total", ctypes.c_ulonglong),
+                            ("virtual_free", ctypes.c_ulonglong), ("extended", ctypes.c_ulonglong)]
+
+            status = _Status()
+            status.length = ctypes.sizeof(_Status)
+            if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
+                return int(status.total)
+            return None
+        return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
+    except (AttributeError, OSError, ValueError):
+        return None
+
+
+def memory_limit() -> int:
+    """The decoding memory budget of this process, in bytes."""
+    global _limit
+    if _limit is None:
+        env = os.environ.get("MICRO_READER_MEMORY_LIMIT")
+        if env:
+            _limit = max(MINIMUM_LIMIT, parse_size(env))
+        else:
+            total = physical_memory()
+            _limit = max(MINIMUM_LIMIT, total // 4) if total else FALLBACK_LIMIT
+    return _limit
+
+
+def set_memory_limit(limit: Union[int, float, str, None]) -> None:
+    """Set the decoding memory budget (bytes or e.g. "8GB"); None restores
+    the default (environment variable, else a quarter of the memory)."""
+    global _limit
+    with _lock:
+        _limit = None if limit is None else max(MINIMUM_LIMIT, parse_size(limit))
+
+
+#: blocks larger than this are streamed where their codec allows: the
+#: decoded-block caches keep blocks up to a quarter of their 64 MB
+STREAM_THRESHOLD = 16 << 20
+
+_reserved = 0
+_peak_reserved = 0
+_room = threading.Condition()
+
+
+def should_stream(nbytes: int) -> bool:
+    """Whether a block of *nbytes* is better streamed than decoded whole
+    (when its codec can be streamed)."""
+    return nbytes > STREAM_THRESHOLD
+
+
+def fits(nbytes: int) -> bool:
+    """Whether a block of *nbytes* can ever be decoded whole."""
+    return nbytes <= memory_limit()
+
+
+class reserve:
+    """``with reserve(nbytes):`` -- hold *nbytes* of the budget while a block
+    is decoded whole; waits while the blocks in flight leave too little."""
+
+    def __init__(self, nbytes: int, what: str = "a block"):
+        self.nbytes, self.what = int(nbytes), what
+
+    def __enter__(self):
+        global _reserved, _peak_reserved
+        if not fits(self.nbytes):
+            raise too_large(self.what, self.nbytes)
+        with _room:
+            while _reserved and _reserved + self.nbytes > memory_limit():
+                _room.wait()
+            _reserved += self.nbytes
+            _peak_reserved = max(_peak_reserved, _reserved)
+        return self
+
+    def __exit__(self, *exc):
+        global _reserved
+        with _room:
+            _reserved -= self.nbytes
+            _room.notify_all()
+
+
+def peak_reserved(reset: bool = False) -> int:
+    """The most bytes reserved at once (for tests and diagnostics)."""
+    global _peak_reserved
+    with _room:
+        peak = _peak_reserved
+        if reset:
+            _peak_reserved = _reserved
+    return peak
+
+
+def too_large(what: str, nbytes: int) -> MemoryLimitError:
+    """The error for a block that cannot be decoded within the budget."""
+    return MemoryLimitError(
+        f"{what} must be decoded whole ({nbytes / 2**20:,.0f} MB), more than the memory "
+        f"limit ({memory_limit() / 2**20:,.0f} MB).  Raise it with "
+        "micro_reader.set_memory_limit(...) or MICRO_READER_MEMORY_LIMIT")

micro_reader/_metadata.py

@@ -0,0 +1,163 @@
+"""Image metadata: the minimal model of docs/METADATA.md.
+
+Pixel sizes per axis, stage position, channel names / colours /
+wavelengths, and the physical value of the image's non-frame indices (the
+wavelength of an emission / excitation index, the time of a lifetime bin) --
+each
+physical value a ``Quantity(value, unit)`` in the file's own unit when
+OME-Zarr has a name for it (D1), ``None`` / absent when the file does not
+say (D2).  Backends fill one ``Metadata`` per image; ``image.metadata``.
+"""
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import dataclass, field, replace
+from typing import NamedTuple, Optional
+
+
+class Quantity(NamedTuple):
+    value: float
+    unit: str
+
+    def to(self, unit: str) -> "Quantity":
+        """This quantity in another unit of the same kind."""
+        if unit == self.unit:
+            return self
+        return Quantity(self.value * _FACTORS[self.unit] / _FACTORS[unit], unit)
+
+
+#: OME-Zarr's units (UDUNITS-2 names), with each one's size in metres or
+#: seconds -- the factors convert between units of one kind.
+SPACE_UNITS = {
+    "angstrom": 1e-10, "attometer": 1e-18, "centimeter": 1e-2, "decimeter": 1e-1,
+    "exameter": 1e18, "femtometer": 1e-15, "foot": 0.3048, "gigameter": 1e9,
+    "hectometer": 1e2, "inch": 0.0254, "kilometer": 1e3, "megameter": 1e6, "meter": 1.0,
+    "micrometer": 1e-6, "mile": 1609.344, "millimeter": 1e-3, "nanometer": 1e-9,
+    "parsec": 3.0857e16, "petameter": 1e15, "picometer": 1e-12, "terameter": 1e12,
+    "yard": 0.9144, "yoctometer": 1e-24, "yottameter": 1e24, "zeptometer": 1e-21,
+    "zettameter": 1e21,
+}
+TIME_UNITS = {
+    "attosecond": 1e-18, "centisecond": 1e-2, "day": 86400.0, "decisecond": 1e-1,
+    "exasecond": 1e18, "femtosecond": 1e-15, "gigasecond": 1e9, "hectosecond": 1e2,
+    "hour": 3600.0, "kilosecond": 1e3, "megasecond": 1e6, "microsecond": 1e-6,
+    "millisecond": 1e-3, "minute": 60.0, "nanosecond": 1e-9, "petasecond": 1e15,
+    "picosecond": 1e-12, "second": 1.0, "terasecond": 1e12, "yoctosecond": 1e-24,
+    "yottasecond": 1e24, "zeptosecond": 1e-21, "zettasecond": 1e21,
+}
+_FACTORS = {**SPACE_UNITS, **TIME_UNITS}
+
+#: how files spell units -> OME-Zarr's name (renaming, not converting)
+_ALIASES = {
+    "m": "meter", "metre": "meter", "meters": "meter", "metres": "meter",
+    "cm": "centimeter", "mm": "millimeter", "um": "micrometer", "µm": "micrometer",
+    "μm": "micrometer", "micron": "micrometer", "microns": "micrometer",
+    "micrometre": "micrometer", "micrometers": "micrometer",
+    "�m": "micrometer", "µm": "micrometer",   # µ decoded with the wrong charset
+    "nm": "nanometer",
+    "nanometre": "nanometer", "pm": "picometer", "a": "angstrom", "å": "angstrom",
+    "Å": "angstrom", "ang": "angstrom", "angstroms": "angstrom", "in": "inch",
+    "inches": "inch", "ft": "foot",
+    "s": "second", "sec": "second", "secs": "second", "seconds": "second",
+    "ms": "millisecond", "msec": "millisecond", "milliseconds": "millisecond",
+    "us": "microsecond", "µs": "microsecond", "μs": "microsecond",
+    "microseconds": "microsecond", "ns": "nanosecond", "ps": "picosecond",
+    "fs": "femtosecond", "min": "minute", "mins": "minute", "minutes": "minute",
+    "h": "hour", "hr": "hour", "hrs": "hour", "hours": "hour", "d": "day", "days": "day",
+}
+
+
+def unit_name(unit: Optional[str]) -> Optional[str]:
+    """OME-Zarr's name for *unit* as a file spells it, or None (unknown)."""
+    if unit is None:
+        return None
+    text = str(unit).strip()
+    if text in _FACTORS:
+        return text
+    if text in _ALIASES:                                  # case matters: "Å", "ms" vs "Ms"
+        return _ALIASES[text]
+    lowered = text.lower()
+    if lowered in _FACTORS:
+        return lowered
+    return _ALIASES.get(lowered)
+
+
+#: lengths in units above this many metres (cm, m, inch, ...) are given in
+#: LARGE_LENGTH_UNIT: microscopy has no use for a pixel of 2.1e-8 meter
+#: (Bugra, 2026-10-02); millimetres and smaller keep the file's unit (D1)
+LARGEST_KEPT_LENGTH = 1e-3
+LARGE_LENGTH_UNIT = "micrometer"
+
+
+def quantity(value, unit: Optional[str], *, positive: bool = True,
+             default_unit: Optional[str] = None) -> Optional[Quantity]:
+    """A Quantity from a file's value and unit spelling; None when the value
+    is missing or not a finite number (or not positive, for sizes), or the
+    unit is unknown.  *default_unit* is used when the file gives none.  The
+    file's unit is kept, except lengths in units larger than a millimetre,
+    which are converted to micrometres."""
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        return None
+    if not math.isfinite(number) or (positive and number <= 0):
+        return None
+    name = unit_name(unit if unit not in (None, "") else default_unit)
+    if name is None:
+        return None
+    if SPACE_UNITS.get(name, 0.0) > LARGEST_KEPT_LENGTH:
+        return Quantity(number, name).to(LARGE_LENGTH_UNIT)
+    return Quantity(number, name)
+
+
+def color_hex(value) -> Optional[str]:
+    """'#RRGGBB' from the ways files store a colour: '#RRGGBB', '#AARRGGBB',
+    'RRGGBB', a signed 32-bit RGBA int (OME), or an (r, g, b) tuple."""
+    if value is None:
+        return None
+    if isinstance(value, (tuple, list)) and len(value) >= 3:
+        r, g, b = (int(round(float(v))) for v in value[:3])
+        if max(r, g, b) <= 1 and any(isinstance(v, float) for v in value[:3]):
+            r, g, b = (int(round(float(v) * 255)) for v in value[:3])
+        return "#{:02X}{:02X}{:02X}".format(r & 255, g & 255, b & 255)
+    if isinstance(value, int):                             # OME: RGBA, signed
+        v = value & 0xFFFFFFFF
+        return "#{:02X}{:02X}{:02X}".format((v >> 24) & 255, (v >> 16) & 255, (v >> 8) & 255)
+    text = str(value).strip().lstrip("#")
+    if re.fullmatch(r"[0-9A-Fa-f]{8}", text):              # AARRGGBB (Zeiss, Imaris)
+        return "#" + text[2:].upper()
+    if re.fullmatch(r"[0-9A-Fa-f]{6}", text):
+        return "#" + text.upper()
+    return None
+
+
+@dataclass(frozen=True)
+class Channel:
+    name: Optional[str] = None
+    color: Optional[str] = None                            # '#RRGGBB'
+    emission_wavelength: Optional[Quantity] = None
+    excitation_wavelength: Optional[Quantity] = None
+
+
+@dataclass(frozen=True)
+class Metadata:
+    """One image's metadata (docs/METADATA.md); what the file does not say
+    is absent / None."""
+    pixel_sizes: dict = field(default_factory=dict)       # axis -> Quantity (x y z t)
+    position: dict = field(default_factory=dict)          # axis -> Quantity
+    channels: tuple = ()                                   # Channel per index of c
+    #: this image's value of its indices that have one (D5): {"emission":
+    #: Quantity(510.0, "nanometer"), "lifetime": Quantity(2.5, "nanosecond")}
+    index_values: dict = field(default_factory=dict)
+
+    def scaled(self, factors: dict) -> "Metadata":
+        """This metadata for a resolution level whose pixels are *factors*
+        (axis -> float) times larger than level 0's."""
+        sizes = {a: Quantity(q.value * factors.get(a, 1.0), q.unit)
+                 for a, q in self.pixel_sizes.items()}
+        return replace(self, pixel_sizes=sizes)
+
+    @classmethod
+    def unknown(cls, n_channels: int = 1) -> "Metadata":
+        return cls(channels=tuple(Channel() for _ in range(max(1, n_channels))))

micro_reader/_pool.py

@@ -0,0 +1,85 @@
+"""One process-wide thread pool for decoding and reading.
+
+Codecs (zlib, LZW, zstd, JPEG, ...) and file reads release the GIL, so a
+single read can decode and read its pieces on all cores instead of only on
+the caller's thread.
+Shared by every scene, so the total number of decoding threads stays fixed
+however many reads run at once.
+
+Size: ``set_decode_threads(n)``, else the ``MICRO_READER_DECODE_THREADS``
+environment variable, else the number of CPUs.  1 decodes on the caller's
+thread (no pool).  Callers that already run many reads in parallel (like
+eubi-bridge's writer) can lower it to avoid oversubscribing the CPU.
+"""
+from __future__ import annotations
+
+import itertools
+import os
+import threading
+from collections import deque
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable, Iterable, Optional
+
+_lock = threading.Lock()
+_pool: Optional[ThreadPoolExecutor] = None
+_threads: Optional[int] = None
+
+
+def decode_threads() -> int:
+    """The number of decoding threads in use."""
+    global _threads
+    if _threads is None:
+        env = os.environ.get("MICRO_READER_DECODE_THREADS")
+        _threads = max(1, int(env)) if env else (os.cpu_count() or 1)
+    return _threads
+
+
+def set_decode_threads(n: int) -> None:
+    """Set the number of decoding threads (1: decode on the caller's thread)."""
+    global _pool, _threads
+    with _lock:
+        old, _pool = _pool, None
+        _threads = max(1, int(n))
+    if old is not None:
+        old.shutdown(wait=False)
+
+
+def _get_pool() -> Optional[ThreadPoolExecutor]:
+    global _pool
+    if decode_threads() < 2:
+        return None
+    with _lock:
+        if _pool is None:
+            _pool = ThreadPoolExecutor(decode_threads(), thread_name_prefix="micro-reader-decode")
+        return _pool
+
+
+def run(tasks: Iterable[Callable[[], int]]) -> int:
+    """Run *tasks* (each returning a count) on the pool; return the sum.
+
+    The iterable is consumed lazily and at most two tasks per thread are in
+    flight, so whatever the tasks hold (compressed bytes, decoded segments)
+    stays bounded however many there are.  Preparation done by the iterable
+    itself -- reading bytes from a file handle that is not thread-safe --
+    happens on the caller's thread.
+    """
+    pool = _get_pool()
+    if pool is None:
+        return sum(task() for task in tasks)
+    tasks = iter(tasks)
+    first, second = next(tasks, None), next(tasks, None)
+    if second is None:                  # nothing to spread: skip the hand-off
+        return first() if first is not None else 0
+    tasks = itertools.chain((first, second), tasks)
+    window, pending, total = 2 * decode_threads(), deque(), 0
+    try:
+        for task in tasks:
+            pending.append(pool.submit(task))
+            if len(pending) >= window:
+                total += pending.popleft().result()
+        while pending:
+            total += pending.popleft().result()
+    finally:
+        for future in pending:          # after an error: don't leave work running
+            future.cancel()
+    return total