opencodecs 0.6.0__tar.gz → 0.7.0__tar.gz

{opencodecs-0.6.0 → opencodecs-0.7.0}/CHANGES.rst

@@ -10,6 +10,48 @@ Versions follow the same ``YYYY.M.D`` cadence as upstream when we
 publish; the entries below cluster work by date rather than by
 release because most of it has shipped continuously to ``main``.
 
+0.7.0 (2026-10-03)
+------------------
+
+``czi_recompress`` takes ``recompress_when``, a predicate over the reader's
+``CziSubBlockEntry``. Sub-blocks it accepts are decoded and re-encoded as
+before; sub-blocks it rejects are carried across BYTE FOR BYTE, keeping their
+original compression. The default stays ``None``, which re-encodes everything,
+so existing callers are unaffected. The returned summary gains
+``recompressed`` and ``carried`` counts.
+
+This exists because a sub-block's compression is not a detail to be
+normalized away. Re-encoding one that is already compressed is wasted work,
+and re-encoding a LOSSY one is destructive: the pixels are decoded from a
+lossy encoding and written back losslessly, so the file grows and the
+artifacts become permanent. Measured on a 505 MB Axioscan slide scan of 481
+JPEG XR sub-blocks:
+
+====================================  ========  ==========  ============
+strategy                              time      output      vs source
+====================================  ========  ==========  ============
+carry JPEG XR across (predicate)      0.36 s    489 MB      0.97x
+re-encode everything (no predicate)   12.6 s    1309 MB     2.59x
+====================================  ========  ==========  ============
+
+So the predicate is not a speed knob. Without it that file more than doubles
+in size, takes 33x longer, and loses the option of ever recovering the
+original encoding. This is what ZEISS's ``czicompress`` expresses with
+``--strategy``, and the three strategies map directly::
+
+    uncompressed        lambda e: e.compression == 0
+    uncompressed+zstd   lambda e: e.compression in (0, 5, 6)
+    all                 None
+
+Supporting this needed a writer that can emit a sub-block it cannot itself
+produce. ``_build_subblock`` gained ``encoded=``, which writes the payload
+exactly as given and records the compression code as-is, with no encode step
+and no look at the pixels. That is the only path by which a JPEG XR sub-block
+can reach a file this writer wrote, and it is why carrying one across cannot
+alter it. Verification is refused on that path: nothing was encoded, so there
+is nothing to check. ``verify=True`` therefore applies only to the sub-blocks
+that were re-encoded.
+
 0.6.0 (2026-10-03)
 ------------------
 

{opencodecs-0.6.0/src/opencodecs.egg-info → opencodecs-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opencodecs
-Version: 0.6.0
+Version: 0.7.0
 Summary: Streaming, network-aware image codecs for scientific imaging (prototype: JPEG XL)
 Author: Kevin Cutler
 License-Expression: BSD-3-Clause

{opencodecs-0.6.0 → opencodecs-0.7.0}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "opencodecs"
-version = "0.6.0"
+version = "0.7.0"
 description = "Streaming, network-aware image codecs for scientific imaging (prototype: JPEG XL)"
 readme = "README.md"
 requires-python = ">=3.10"

{opencodecs-0.6.0 → opencodecs-0.7.0}/src/opencodecs/__init__.py

@@ -349,4 +349,4 @@ __all__ = [
     "czi_recompress", "subblock_dims",
 ]
 
-__version__ = "0.6.0"
+__version__ = "0.7.0"

{opencodecs-0.6.0 → opencodecs-0.7.0}/src/opencodecs/_czi_writer.py

@@ -74,6 +74,30 @@ class CziWriterError(RuntimeError):
     """Raised on writer state-machine violations."""
 
 
+class _CarriedSubBlock:
+    """A sub-block copied across unchanged, not re-encoded.
+
+    Carries the encoded payload and the compression code it was written
+    with, plus a zero-filled array of the right shape and dtype so the
+    existing header path can read shape/dtype without the pixels ever being
+    decoded. ``np.empty`` would do, but zeros make an accidental use of
+    these pixels obvious rather than random.
+    """
+
+    __slots__ = ("payload", "compression", "dims", "pyramid_type",
+                 "shape_array", "logical_shape", "pixel_type")
+
+    def __init__(self, payload, *, compression, dims, pyramid_type,
+                 shape_array, logical_shape, pixel_type):
+        self.payload = payload
+        self.compression = compression
+        self.pixel_type = pixel_type
+        self.dims = dims
+        self.pyramid_type = pyramid_type
+        self.shape_array = shape_array
+        self.logical_shape = logical_shape
+
+
 class CziCancelled(CziWriterError):
     """Raised when a caller's ``should_continue`` stops a rewrite.
 
@@ -339,6 +363,7 @@ def _build_subblock(
     dims: tuple | None = None,
     verification_input: list | None = None,
     scratch=None,
+    encoded: object | None = None,
 ) -> tuple[_SubBlockSegment, dict]:
     """Build one ZISRAWSUBBLOCK segment as parts.
 
@@ -351,7 +376,21 @@ def _build_subblock(
     check runs after this returns while the producer may already be
     refilling ``array``. Without it nothing here outlives the call, so the
     array's own buffer feeds the compressor or becomes the raw payload.
+
+    With ``encoded`` the payload is written exactly as given and
+    ``compression_code`` is recorded as-is, with no encode step and no look
+    at the pixels. That is how a sub-block is carried across unchanged, and
+    it is the only path that can emit a compression this writer cannot
+    produce (JPEG XR, for instance). ``array`` is then used only for its
+    shape and dtype, so a zero-filled array of the right shape is enough.
+    Verification is not available on this path: there is nothing to check,
+    because nothing was encoded.
     """
+    if encoded is not None and verification_input is not None:
+        raise CziWriterError(
+            "czi: verification_input cannot be combined with encoded; a "
+            "carried-across sub-block is not re-encoded, so there is "
+            "nothing to verify")
     h, w = array.shape[:2]
     logical_h, logical_w = logical_shape
     start_y, start_x = location
@@ -390,7 +429,10 @@ def _build_subblock(
         pixels = memoryview(contiguous).cast("B")
         keep = contiguous
     itemsize = array.dtype.itemsize
-    if compression_code == 0:
+    if encoded is not None:
+        data = encoded
+        keep = encoded
+    elif compression_code == 0:
         data = pixels
     elif compression_code == 5:
         from .codecs._zstd import encode as zstd_encode
@@ -687,6 +729,17 @@ class _CziStreamWriter(Writer):
             workers = resolve_workers(None, 1 << 30, output_bytes=None, max_workers=8)
         workers = max(1, int(workers))
 
+        def item_bytes(item):
+            """Cost of one snapshotted item, for batching and reservation.
+
+            A carried sub-block has no array: its cost is the encoded payload
+            it already owns. Both batching sites go through here so they
+            cannot disagree about what an item weighs.
+            """
+            if isinstance(item, _CarriedSubBlock):
+                return len(item.payload)
+            return item[0].nbytes
+
         def snapshot(frame):
             """Own the frame's pixels before the producer can touch them.
 
@@ -695,6 +748,10 @@ class _CziStreamWriter(Writer):
             it is a down-scaled level. Those forms are what make a faithful
             re-encode possible on this path as well as on ``write_frame``.
             """
+            if isinstance(frame, _CarriedSubBlock):
+                # Nothing to snapshot: the payload is already bytes we own,
+                # and the pixels are never looked at.
+                return frame
             dims = override_pyramid = None
             if isinstance(frame, tuple):
                 if len(frame) == 2:
@@ -728,7 +785,7 @@ class _CziStreamWriter(Writer):
             batch, batch_bytes = [], 0
             for frame in frames:
                 item = snapshot(frame)
-                n = item[0].nbytes
+                n = item_bytes(item)
                 if batch and batch_bytes + n > batch_limit:
                     yield batch
                     batch, batch_bytes = [], 0
@@ -738,7 +795,7 @@ class _CziStreamWriter(Writer):
                 yield batch
 
         def size(batch):
-            n = sum(item[0].nbytes for item in batch)
+            n = sum(item_bytes(item) for item in batch)
             # Snapshot + encoded output (bounded by the input for these
             # codecs) + decoded verification result and comparison scratch.
             return 2 * n + (verification_reservation(n) if verify else 0)
@@ -746,7 +803,19 @@ class _CziStreamWriter(Writer):
         def encode(batch):
             out = []
             scratch = _worker_scratch()
-            for pixels, logical_shape, pyramid_type, dims in batch:
+            for item in batch:
+                if isinstance(item, _CarriedSubBlock):
+                    segment, entry = _build_subblock(
+                        item.shape_array,
+                        pixel_type=item.pixel_type,
+                        compression_code=item.compression, hilo=self._hilo,
+                        file_position=0, logical_shape=item.logical_shape,
+                        pyramid_type=item.pyramid_type, dims=item.dims,
+                        verification_input=None, scratch=None,
+                        encoded=item.payload)
+                    out.append((segment, entry))
+                    continue
+                pixels, logical_shape, pyramid_type, dims = item
                 captured = [] if verify else None
                 segment, entry = _build_subblock(
                     pixels, pixel_type=_DTYPE_TO_PIXELTYPE[pixels.dtype],
@@ -992,7 +1061,8 @@ __all__ = ["CziWriter", "CziPyramidWriter", "CziWriterError"]
 def czi_recompress(src, dst, *, compression: str = "zstdhdr", hilo: bool = True,
                workers=None, verify: bool = False,
                background_encode: bool = False,
-               should_continue=None) -> dict:
+               should_continue=None,
+               recompress_when=None) -> dict:
     """Rewrite a CZI with different compression, keeping the container.
 
     Every sub-block keeps its dimension coordinates, its scene and mosaic
@@ -1016,8 +1086,30 @@ def czi_recompress(src, dst, *, compression: str = "zstdhdr", hilo: bool = True,
     know which. Encodes already dispatched still finish, so cancellation
     takes effect within roughly ``workers`` sub-blocks rather than instantly.
 
-    Returns a summary dict: sub-block count, input and output sizes, and the
-    compression that was applied.
+    ``recompress_when`` is an optional predicate over the reader's
+    ``CziSubBlockEntry``. Sub-blocks it accepts are decoded and re-encoded;
+    sub-blocks it rejects are carried across BYTE FOR BYTE, keeping their
+    original compression. Default ``None`` re-encodes everything, which is
+    what earlier versions did.
+
+    This is what makes the selective strategies of ZEISS's ``czicompress``
+    expressible, and the reason they exist: re-encoding an already-compressed
+    sub-block is at best wasted work, and for a lossy one it is destructive.
+    A JPEG XR sub-block decoded and re-encoded as lossless zstdhdr grows,
+    and bakes the lossy artifacts in permanently. Carrying it across cannot
+    do that.
+
+        uncompressed only   lambda e: e.compression == 0
+        uncompressed+zstd   lambda e: e.compression in (0, 5, 6)
+        everything          None
+
+    ``verify=True`` applies only to the sub-blocks that were re-encoded; a
+    carried-across payload is unchanged by definition, so there is nothing
+    to check.
+
+    Returns a summary dict: sub-block count, input and output sizes, the
+    compression that was applied, and how many sub-blocks were re-encoded
+    versus carried across.
     """
     import os
 
@@ -1029,6 +1121,8 @@ def czi_recompress(src, dst, *, compression: str = "zstdhdr", hilo: bool = True,
             raise CziWriterError(f"czi: {src} has no sub-blocks to recompress")
         metadata = reader.metadata_bytes or b"<Metadata/>"
 
+        counts = {"recompressed": 0, "carried": 0}
+
         def frames():
             # One sub-block at a time: a slide scan does not fit in memory,
             # and write_many only needs the next frame, not all of them.
@@ -1038,7 +1132,30 @@ def czi_recompress(src, dst, *, compression: str = "zstdhdr", hilo: bool = True,
                         f"czi: recompress of {src} cancelled after "
                         f"{i} of {n} sub-blocks")
                 entry = reader.entries[i]
-                yield reader[i], subblock_dims(entry), entry.pyramid_type
+                if recompress_when is None or recompress_when(entry):
+                    counts["recompressed"] += 1
+                    yield reader[i], subblock_dims(entry), entry.pyramid_type
+                else:
+                    # Carried across: the encoded bytes are copied straight
+                    # from the source and the original compression is kept,
+                    # so the pixels are never decoded and cannot change.
+                    counts["carried"] += 1
+                    d = subblock_dims(entry)
+                    axes = {name.rstrip(b"\x00"): (size, stored)
+                            for name, _start, size, _co, stored in d}
+                    log_h, st_h = axes[b"Y"]
+                    log_w, st_w = axes[b"X"]
+                    with reader._payload(entry) as view:
+                        # Copy: the reader hands back a buffer it reuses, so
+                        # a borrowed view would be overwritten by the next
+                        # sub-block before the writer emits this one.
+                        payload = bytes(view)
+                    yield _CarriedSubBlock(
+                        payload, compression=entry.compression,
+                        dims=d, pyramid_type=entry.pyramid_type,
+                        shape_array=np.zeros((st_h, st_w), entry.dtype),
+                        logical_shape=(log_h, log_w),
+                        pixel_type=entry.pixel_type)
 
         try:
             with CziWriter(str(dst), compression=compression, hilo=hilo,
@@ -1061,4 +1178,6 @@ def czi_recompress(src, dst, *, compression: str = "zstdhdr", hilo: bool = True,
         "src_bytes": os.path.getsize(src),
         "dst_bytes": os.path.getsize(dst),
         "compression": compression,
+        "recompressed": counts["recompressed"],
+        "carried": counts["carried"],
     }

{opencodecs-0.6.0 → opencodecs-0.7.0/src/opencodecs.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opencodecs
-Version: 0.6.0
+Version: 0.7.0
 Summary: Streaming, network-aware image codecs for scientific imaging (prototype: JPEG XL)
 Author: Kevin Cutler
 License-Expression: BSD-3-Clause

{opencodecs-0.6.0 → opencodecs-0.7.0}/tests/test_czi_recompress.py

@@ -351,3 +351,112 @@ def test_a_failing_predicate_also_leaves_no_output(tmp_path):
     with pytest.raises(ValueError):
         czi_recompress(src, dst, workers=PARTIAL_WORKERS, should_continue=boom)
     assert not dst.exists()
+
+
+# ------------------------------------------- selective recompression
+# `recompress_when` exists so the selective strategies of ZEISS's czicompress
+# are expressible. The point is NOT speed: re-encoding an already-compressed
+# sub-block is wasted work, and for a LOSSY one it is destructive, because a
+# JPEG XR block decoded and re-encoded as lossless zstdhdr grows and bakes the
+# lossy artifacts in permanently.
+#
+# Every assertion here is on the OUTPUT'S COMPRESSION CODES, not just its
+# pixels. A pass-through that quietly re-encoded would still produce a
+# readable file with identical pixels, so comparing arrays cannot catch it.
+
+def _mixed_src(tmp_path):
+    """Two uncompressed sub-blocks and two already-zstd ones."""
+    raw = [(_tile(64, 64, 200 + i), (0, i * 64), [(b"C", i)]) for i in range(2)]
+    src_u = tmp_path / "u.czi"
+    src_u.write_bytes(mosaic_czi_bytes(raw, compression=0))
+    pre = tmp_path / "pre.czi"
+    czi_recompress(src_u, pre, compression="zstdhdr")
+    return src_u, pre
+
+
+def _codes(path):
+    with CziReader(str(path)) as r:
+        return [e.compression for e in r.entries]
+
+
+def test_default_still_recompresses_everything(tmp_path):
+    src, _ = _mixed_src(tmp_path)
+    dst = tmp_path / "d.czi"
+    info = czi_recompress(src, dst, compression="zstdhdr")
+    assert info["recompressed"] == 2 and info["carried"] == 0
+    assert set(_codes(dst)) == {6}
+
+
+def test_uncompressed_only_leaves_compressed_blocks_alone(tmp_path):
+    """The czicompress default: only code 0 is touched."""
+    _, pre = _mixed_src(tmp_path)
+    assert set(_codes(pre)) == {6}, "fixture is not already compressed"
+    dst = tmp_path / "d.czi"
+    info = czi_recompress(pre, dst, compression="zstdhdr",
+                          recompress_when=lambda e: e.compression == 0)
+    assert info["recompressed"] == 0, "re-encoded a block it was told to skip"
+    assert info["carried"] == 2
+    assert _codes(dst) == _codes(pre)
+    _assert_faithful(pre, dst)
+
+
+def test_carried_payload_is_byte_identical(tmp_path):
+    """Carried means copied, so the encoded bytes must match exactly."""
+    _, pre = _mixed_src(tmp_path)
+    dst = tmp_path / "d.czi"
+    czi_recompress(pre, dst, compression="none",
+                   recompress_when=lambda e: False)
+    with CziReader(str(pre)) as a, CziReader(str(dst)) as b:
+        for ea, eb in zip(a.entries, b.entries):
+            with a._payload(ea) as pa, b._payload(eb) as pb:
+                assert bytes(pa) == bytes(pb), "carried payload was altered"
+            assert ea.compression == eb.compression
+
+
+def test_carried_blocks_keep_their_code_while_others_change(tmp_path):
+    """A mixed file: some blocks re-encoded, some carried, in one pass."""
+    _, pre = _mixed_src(tmp_path)
+    extra = [(_tile(64, 64, 300 + i), (0, (i + 2) * 64), [(b"C", i + 2)])
+             for i in range(2)]
+    mixed = tmp_path / "mixed.czi"
+    mixed.write_bytes(mosaic_czi_bytes(extra, compression=0))
+    dst = tmp_path / "d.czi"
+    info = czi_recompress(mixed, dst, compression="zstdhdr",
+                          recompress_when=lambda e: e.compression == 0)
+    assert info["recompressed"] == 2 and info["carried"] == 0
+    assert set(_codes(dst)) == {6}
+
+
+def test_predicate_sees_a_real_entry(tmp_path):
+    """The predicate gets the reader's entry, not an opaque index."""
+    src, _ = _mixed_src(tmp_path)
+    seen = []
+
+    def pred(e):
+        seen.append((e.compression, e.pixel_type, e.pyramid_type))
+        return True
+
+    czi_recompress(src, tmp_path / "d.czi", recompress_when=pred)
+    assert len(seen) == 2
+    assert all(c == 0 for c, _p, _y in seen)
+
+
+def test_verify_applies_only_to_recompressed_blocks(tmp_path):
+    """verify=True must not choke on a carried block it never encoded."""
+    _, pre = _mixed_src(tmp_path)
+    dst = tmp_path / "d.czi"
+    info = czi_recompress(pre, dst, compression="zstdhdr", verify=True,
+                          recompress_when=lambda e: e.compression == 0)
+    assert info["carried"] == 2
+    _assert_faithful(pre, dst)
+
+
+def test_encoded_and_verification_input_are_refused():
+    """Verifying a carried block is meaningless; the writer says so."""
+    import numpy as np
+    from opencodecs._czi_writer import _build_subblock
+    with pytest.raises(CziWriterError, match="nothing to verify"):
+        _build_subblock(np.zeros((4, 4), "uint16"), pixel_type=1,
+                        compression_code=0, hilo=False, file_position=0,
+                        logical_shape=(4, 4), verification_input=[],
+                        encoded=b"x")