thyra 3.7.4__tar.gz → 3.7.5__tar.gz

{thyra-3.7.4 → thyra-3.7.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: thyra
-Version: 3.7.4
+Version: 3.7.5
 Summary: A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics
 Project-URL: Homepage, https://github.com/M4i-Imaging-Mass-Spectrometry/thyra
 Project-URL: Repository, https://github.com/M4i-Imaging-Mass-Spectrometry/thyra
@@ -47,7 +47,7 @@ Requires-Dist: spatialdata<0.9,>=0.8.0
 Requires-Dist: tifffile>=2022.8.12
 Requires-Dist: tqdm>=4.50.0
 Requires-Dist: xarray>=2024.10.0
-Requires-Dist: zarr<3.2,>=3.1.4
+Requires-Dist: zarr<3.2,>=3.1.6
 Description-Content-Type: text/markdown
 
 <p align="center">

{thyra-3.7.4 → thyra-3.7.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "thyra"
-version = "3.7.4"
+version = "3.7.5"
 description = "A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics"
 authors = [{ name = "Theodoros Visvikis", email = "t.visvikis@maastrichtuniversity.nl" }]
 maintainers = [{ name = "Theodoros Visvikis", email = "t.visvikis@maastrichtuniversity.nl" }]
@@ -91,16 +91,28 @@ dependencies = [
     # bit ome_zarr 0.12.x with newer dask, and spatialdata 0.8 requires >= 0.16.
     "ome-zarr>=0.16.0, <0.19",        # tested 0.18.0
     "tqdm>=4.50.0",
-    # Floor is 3.1.4, and it is load-bearing. anndata >= 0.13 writes every zarr v3
-    # array with `shards="auto"`. zarr < 3.1.4 sized the auto chunk with
-    # `max_bytes=1024` where its own comment says "aim for a 1MiB chunk" (compare
-    # `_auto_partition` in 3.1.3 vs 3.1.4), so one shard -- one FILE -- came out at
-    # ~1 KB. Measured on test_data/pea.imzML: 393,310 files for a single table and
-    # 434 s of a 445 s conversion spent in the write, against 12 files and 2.7 s on
-    # 3.1.6. Nothing in Thyra changed; the anndata 0.13 bump alone did this.
-    # 3.1.4 is also where `array.target_shard_size_bytes` arrives, which is the
-    # knob _chunking.table_write_config sets to claim the budget from anndata.
-    "zarr>=3.1.4, <3.2",              # tested 3.1.6
+    # Floor is 3.1.6, and it is load-bearing. anndata >= 0.13 writes every zarr v3
+    # array with `shards="auto"`, and `_auto_partition` sized the auto INNER chunk
+    # with `max_bytes=1024` where the line right above it says "aim for a 1MiB
+    # chunk". zarr-python#3603 changed that 1024 to 1048576; it merged six days
+    # AFTER 3.1.5 shipped, so 3.1.4 and 3.1.5 both still carry the defect
+    # (`chunk_grids.py:267` in each) and 3.1.6 is the first release without it.
+    # Do not drop the floor to 3.1.4 on the grounds that 3.1.4 is where
+    # `array.target_shard_size_bytes` arrives -- the knob _chunking.table_write_config
+    # sets to claim the budget from anndata. That knob is necessary, not sufficient.
+    #
+    # Both halves of the damage, measured on X/data of test_data/pea.imzML (60.1M
+    # nnz, f8, 481 MB):
+    #   - Below 3.1.4 the knob does not exist, so a shard is 2 auto chunks -- ~1.8 KB,
+    #     and a shard is one FILE. 393,310 files for a single table, and 434 s of a
+    #     445 s conversion spent in the write, against 12 files and 2.7 s on 3.1.6.
+    #     Nothing in Thyra changed; the anndata 0.13 bump alone did this.
+    #   - On 3.1.4/3.1.5 the 128 MiB budget IS honoured, so the file count stays sane
+    #     (4), but the chunk inside those files is 920 B: ~145,900 separately
+    #     compressed chunks per shard against 142 on 3.1.6, plus the shard index that
+    #     implies. A 3.1.5 resolve buys that silently, and the shard-level assertion
+    #     cannot see it; _chunking.MIN_TABLE_CHUNK_BYTES is what catches it.
+    "zarr>=3.1.6, <3.2",              # tested 3.1.6
     "imagecodecs>=2024.1.1",  # For reading compressed TIFF optical images
     # Floor is 0.13.2, not 0.13.0. anndata 0.13.0 imports
     # `typing_extensions.sentinel` (needs typing-extensions >= 4.16) but declares

{thyra-3.7.4 → thyra-3.7.5}/thyra/__init__.py

@@ -36,7 +36,7 @@ warnings.filterwarnings(
     category=FutureWarning,
 )
 
-__version__ = "3.7.4"
+__version__ = "3.7.5"
 
 # Import key components - avoid wildcard imports
 try:

{thyra-3.7.4 → thyra-3.7.5}/thyra/converters/spatialdata/_chunking.py

@@ -53,13 +53,28 @@ INNER_TILE_EDGE = 512
 # began auto-sharding, so read amplification is unchanged.
 TABLE_SHARD_TARGET_BYTES = 128 * 1024 * 1024
 
-# zarr < 3.1.4 computed the auto chunk with ``max_bytes=1024`` where 1 MiB was
-# meant, so ``shards="auto"`` produced ~1 KB shards -- one file per KB of table.
-# It also predates ``array.target_shard_size_bytes``, making the budget above
-# unenforceable. pyproject pins the floor at 3.1.4 for exactly this reason;
-# this constant is what the regression test checks the floor still buys us.
+# zarr < 3.1.6 computed the auto chunk with ``max_bytes=1024`` where 1 MiB was
+# meant (zarr-python#3603, first released in 3.1.6 -- it landed after both 3.1.4
+# and 3.1.5). zarr < 3.1.4 additionally predates ``array.target_shard_size_bytes``,
+# making the budget above unenforceable, and there ``shards="auto"`` produced
+# ~1 KB SHARDS -- one file per KB of table. pyproject pins the floor at 3.1.6 for
+# both reasons.
+#
+# This constant guards the shard, i.e. the file, so it only ever catches that
+# second, pre-3.1.4 half: on 3.1.4/3.1.5 the budget above still lands large
+# shards and it is the chunk inside them that collapses. MIN_TABLE_CHUNK_BYTES
+# is the half that catches those two.
 MIN_TABLE_SHARD_BYTES = 64 * 1024
 
+# Minimum size for the INNER chunk of a table array -- the unit that sits inside
+# a shard and the unit a random read decompresses. This is the half of the defect
+# a large shard hides: on 3.1.4/3.1.5 the file count looks fine while each file
+# quietly holds ~146,000 separately compressed chunks instead of ~142, and pays
+# for all of them in the shard index. 64 KiB sits well above the ~800 B those
+# versions hand the regression-test fixture and well below the ~200 KB 3.1.6
+# hands it, so the two are never close to the threshold.
+MIN_TABLE_CHUNK_BYTES = 64 * 1024
+
 
 @contextmanager
 def table_write_config() -> Iterator[None]: