thyra 2.2.2__tar.gz → 2.2.4__tar.gz

{thyra-2.2.2 → thyra-2.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thyra
-Version: 2.2.2
+Version: 2.2.4
 Summary: A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics
 License: MIT
 License-File: LICENSE
@@ -39,7 +39,7 @@ Requires-Dist: pyimzML (>=1.4.0)
 Requires-Dist: scipy (>=1.7.0)
 Requires-Dist: spatialdata (>=0.8.0,<0.9)
 Requires-Dist: tqdm (>=4.50.0)
-Requires-Dist: zarr (>=3.0.0,<3.2)
+Requires-Dist: zarr (>=3.1.4,<3.2)
 Project-URL: Bug Tracker, https://github.com/M4i-Imaging-Mass-Spectrometry/thyra/issues
 Project-URL: Changelog, https://github.com/M4i-Imaging-Mass-Spectrometry/thyra/blob/main/CHANGELOG.md
 Project-URL: Contributing, https://github.com/M4i-Imaging-Mass-Spectrometry/thyra/blob/main/CONTRIBUTING.md

{thyra-2.2.2 → thyra-2.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "thyra"
-version = "2.2.2"
+version = "2.2.4"
 description = "A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics"
 authors = ["Theodoros Visvikis <t.visvikis@maastrichtuniversity.nl>"]
 maintainers = ["Theodoros Visvikis <t.visvikis@maastrichtuniversity.nl>"]
@@ -86,7 +86,16 @@ spatialdata = ">=0.8.0, <0.9"       # tested 0.8.0
 # bit ome_zarr 0.12.x with newer dask, and spatialdata 0.8 requires >= 0.16.
 ome-zarr = ">=0.16.0, <0.19"        # tested 0.18.0
 tqdm = ">=4.50.0"
-zarr = ">=3.0.0, <3.2"              # tested 3.1.3
+# Floor is 3.1.4, and it is load-bearing. anndata >= 0.13 writes every zarr v3
+# array with `shards="auto"`. zarr < 3.1.4 sized the auto chunk with
+# `max_bytes=1024` where its own comment says "aim for a 1MiB chunk" (compare
+# `_auto_partition` in 3.1.3 vs 3.1.4), so one shard -- one FILE -- came out at
+# ~1 KB. Measured on test_data/pea.imzML: 393,310 files for a single table and
+# 434 s of a 445 s conversion spent in the write, against 12 files and 2.7 s on
+# 3.1.6. Nothing in Thyra changed; the anndata 0.13 bump alone did this.
+# 3.1.4 is also where `array.target_shard_size_bytes` arrives, which is the
+# knob _chunking.table_write_config sets to claim the budget from anndata.
+zarr = ">=3.1.4, <3.2"              # tested 3.1.6
 imagecodecs = ">=2024.1.1"  # For reading compressed TIFF optical images
 # Floor is 0.13.2, not 0.13.0. anndata 0.13.0 imports
 # `typing_extensions.sentinel` (needs typing-extensions >= 4.16) but declares

{thyra-2.2.2 → thyra-2.2.4}/thyra/__init__.py

@@ -36,7 +36,7 @@ warnings.filterwarnings(
     category=FutureWarning,
 )
 
-__version__ = "2.2.2"
+__version__ = "2.2.4"
 
 # Import key components - avoid wildcard imports
 try:

thyra-2.2.4/thyra/converters/spatialdata/_chunking.py

@@ -0,0 +1,89 @@
+"""Chunk/shard policy for the zarr Thyra writes: rasters and the MSI table.
+
+Single source of truth for how Thyra chunks the raster images it writes to
+SpatialData zarr -- and the designated seam for tile-aligned zarr v3 sharding
+once scverse/spatialdata PR #1106 (``raster_write_kwargs``) lands.
+
+Cross-repo contract: the viewer (Ousia) morphology tile server streams
+``DEFAULT_TILE_SIZE`` (= 512) edge tiles. Once sharding lands, the zarr INNER
+chunk edge must equal that so one served tile == one decompress. Thyra is a
+separate package and CANNOT import Ousia, so the coupling is DUPLICATED BY
+CONTRACT here -- keep ``INNER_TILE_EDGE`` numerically in sync with Ousia's
+``DEFAULT_TILE_SIZE``; a silent divergence reintroduces cold-tile amplification
+on the viewer. See the Ousia ADR docs/ADR-pyramid-tile-sharding.md.
+
+The raster policy above is passed per-array. The TABLE policy below cannot be:
+``SpatialData.write`` hands the table straight to ``anndata.write_elem`` and
+exposes no ``dataset_kwargs`` seam, so the only supported lever is zarr's own
+config. ``table_write_config()`` is that lever, and every writer that reaches
+zarr through spatialdata (the 2D, 3D and streaming-COO converters, all via
+``_save_output``) must hold it open across the write. The streaming-PCS writer
+hand-rolls its ``create_array`` calls and sizes them itself.
+"""
+
+from contextlib import contextmanager
+from typing import Iterator, Tuple
+
+import zarr
+
+# Outer chunk edge used TODAY (pre-sharding the chunk IS the outer block): a 4096
+# block means a viewer 512-tile read decompresses at most one chunk per request.
+IMAGE_CHUNK_EDGE = 4096
+
+# Future zarr INNER chunk edge once #1106 lands == Ousia DEFAULT_TILE_SIZE (512).
+# Duplicated by contract (Thyra cannot import Ousia).
+INNER_TILE_EDGE = 512
+
+# Uncompressed byte budget for ONE SHARD of a table array -- X/data, X/indices,
+# X/indptr and every obs/var column anndata writes. One shard is one FILE, so
+# this is what keeps a 36M-nnz table (test_data/bellini.imzML) in 11 files
+# instead of 394,699.
+#
+# anndata >= 0.13 turns on ``shards="auto"`` for zarr v3 and would otherwise
+# pick its own 1 GB budget; it explicitly yields to a caller-set value (see
+# ``anndata._io.specs.methods.zarr_v3_sharding``). We set a smaller one because
+# a shard is assembled in memory before it is written -- anndata's own comment
+# says so -- and Thyra converts multi-GB acquisitions on workstations, where a
+# 1 GB write buffer is a real cost. 128 MiB is still enormous relative to the
+# file counts that hurt: test_data/pea.imzML (60.1M nnz, 481 MB of X/data)
+# lands in 4 data files, so even a table 100x that size stays in the hundreds.
+#
+# The INNER chunk stays zarr's ~1 MiB auto pick: that is the unit a random read
+# decompresses, and it is the size the non-sharded path used before anndata
+# began auto-sharding, so read amplification is unchanged.
+TABLE_SHARD_TARGET_BYTES = 128 * 1024 * 1024
+
+# zarr < 3.1.4 computed the auto chunk with ``max_bytes=1024`` where 1 MiB was
+# meant, so ``shards="auto"`` produced ~1 KB shards -- one file per KB of table.
+# It also predates ``array.target_shard_size_bytes``, making the budget above
+# unenforceable. pyproject pins the floor at 3.1.4 for exactly this reason;
+# this constant is what the regression test checks the floor still buys us.
+MIN_TABLE_SHARD_BYTES = 64 * 1024
+
+
+@contextmanager
+def table_write_config() -> Iterator[None]:
+    """Hold Thyra's table shard budget open across a ``SpatialData.write``.
+
+    Wraps zarr's global config, so it must stay open for the *whole* write --
+    the arrays are created lazily inside anndata, not up front.
+    """
+    with zarr.config.set({"array.target_shard_size_bytes": TABLE_SHARD_TARGET_BYTES}):
+        yield
+
+
+def image_chunks(spatial_ndim: int, edge: int = IMAGE_CHUNK_EDGE) -> Tuple[int, ...]:
+    """The zarr chunk tuple for a Thyra-written raster image (leading channel axis).
+
+    ``spatial_ndim == 2`` (c, y, x) -> ``(1, edge, edge)``;
+    ``spatial_ndim == 3`` (c, z, y, x) -> ``(1, 1, edge, edge)``.
+
+    This is the ONE place the future inner=``INNER_TILE_EDGE`` + outer-shard policy
+    will live when spatialdata PR #1106 ships ``raster_write_kwargs``. Today it
+    returns the pre-sharding behaviour, byte-for-byte.
+    """
+    if spatial_ndim == 2:
+        return (1, edge, edge)
+    if spatial_ndim == 3:
+        return (1, 1, edge, edge)
+    raise ValueError(f"spatial_ndim must be 2 or 3, got {spatial_ndim}")

{thyra-2.2.2 → thyra-2.2.4}/thyra/converters/spatialdata/base_spatialdata_converter.py

@@ -21,7 +21,7 @@ from ...resampling.gaps import zero_across_gaps
 from ...resampling.tic import preserved_tic, rescale_to_preserved_tic
 from ...resampling.types import ResamplingConfig
 from ...utils.zarr_atomic_write import install_windows_atomic_write_retry
-from ._chunking import image_chunks
+from ._chunking import image_chunks, table_write_config
 
 logger = logging.getLogger(__name__)
 
@@ -1921,8 +1921,12 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
             # Add metadata
             self.add_metadata(sdata)
 
-            # Write to disk
-            with _suppress_upstream_warnings():
+            # Write to disk. table_write_config() must wrap the write itself,
+            # not just the SpatialData construction: anndata creates the table's
+            # zarr arrays lazily inside sdata.write, and that is where the shard
+            # budget is read. Without it the table lands in ~1 KB shards -- one
+            # file per KB -- and the write dominates conversion wall-clock.
+            with _suppress_upstream_warnings(), table_write_config():
                 sdata.write(str(self.output_path))
                 zarr.consolidate_metadata(str(self.output_path))
             logger.info(f"Successfully saved SpatialData to {self.output_path}")

{thyra-2.2.2 → thyra-2.2.4}/thyra/metadata/extractors/imzml_extractor.py

@@ -9,7 +9,7 @@ from numpy.typing import NDArray
 from pyimzml.ImzMLParser import ImzMLParser
 
 from ...core.base_extractor import MetadataExtractor
-from ...resampling.constants import ImzMLAccessions, SpectrumType
+from ...resampling.constants import BinaryDataType, ImzMLAccessions, SpectrumType
 from ..types import ComprehensiveMetadata, EssentialMetadata
 
 logger = logging.getLogger(__name__)
@@ -131,7 +131,7 @@ class ImzMLMetadataExtractor(MetadataExtractor):
             if param_by_name is None or not isinstance(param_by_name, dict):
                 return False
 
-            return "continuous" in param_by_name
+            return BinaryDataType.CONTINUOUS in param_by_name
         except Exception:
             return False
 
@@ -522,19 +522,27 @@ class ImzMLMetadataExtractor(MetadataExtractor):
             return None
 
     def _detect_centroid_spectrum(self) -> Optional[str]:
-        """Detect spectrum type by looking for MS:1000127 (centroid) or MS:1000128 (profile)."""
+        """Detect spectrum type by looking for MS:1000127 (centroid) or MS:1000128 (profile).
+
+        What the file *declares* wins, wherever it is written. Only when the
+        file declares nothing does Thyra guess -- see
+        :meth:`_guess_spectrum_type_from_storage_mode` for what that guess is
+        worth.
+        """
         try:
-            # Method 1: Check parser metadata first (no XML parsing needed)
-            result = self._check_parser_metadata_for_centroid()
+            # 1. The declaration, from metadata the parser already holds.
+            result = self._check_parser_metadata_for_spectrum_type()
             if result:
                 return result
 
-            # Method 2: Stream-parse XML for spectrum type markers (memory efficient)
+            # 2. The same declaration, wherever else in the document it sits.
+            #    Streams the XML, so it is tried only if step 1 came up empty.
             result = self._check_xml_for_spectrum_type()
             if result:
                 return result
 
-            return None
+            # 3. Nothing declared. Only now is a guess appropriate.
+            return self._guess_spectrum_type_from_storage_mode()
         except Exception as e:
             logger.debug(f"Could not detect spectrum type: {e}")
             return None
@@ -607,8 +615,13 @@ class ImzMLMetadataExtractor(MetadataExtractor):
             logger.warning("defusedxml not available, using xml.etree.ElementTree")
             return ET
 
-    def _check_parser_metadata_for_centroid(self) -> Optional[str]:
-        """Check parser metadata for processed flag indicating centroid data."""
+    def _file_description_params(self) -> Optional[Dict[str, Any]]:
+        """Return the fileDescription cvParams the parser already parsed.
+
+        ``param_by_name`` maps a term's CV *name* to its parsed value, and a
+        valueless cvParam -- which is what a flag like ``profile spectrum``
+        is -- parses to ``True``.
+        """
         if not (hasattr(self.parser, "metadata") and self.parser.metadata):
             return None
 
@@ -616,13 +629,70 @@ class ImzMLMetadataExtractor(MetadataExtractor):
             return None
 
         file_desc = self.parser.metadata.file_description
-        if not hasattr(file_desc, "param_by_name"):
+        params = getattr(file_desc, "param_by_name", None)
+        if not isinstance(params, dict):
+            return None
+
+        return params
+
+    def _check_parser_metadata_for_spectrum_type(self) -> Optional[str]:
+        """Read the declared spectrum representation out of the fileDescription.
+
+        Cheap: the parser has already read this, so no XML pass is needed.
+        ``SpectrumType.CENTROID`` and ``SpectrumType.PROFILE`` are the CV
+        names of ``MS:1000127`` and ``MS:1000128``, which is what
+        ``param_by_name`` is keyed on.
+        """
+        params = self._file_description_params()
+        if params is None:
+            return None
+
+        if params.get(SpectrumType.CENTROID, False):
+            logger.info(
+                f"Detected centroid spectrum from declared "
+                f"{ImzMLAccessions.CENTROID_SPECTRUM} in fileDescription"
+            )
+            return SpectrumType.CENTROID
+
+        if params.get(SpectrumType.PROFILE, False):
+            logger.info(
+                f"Detected profile spectrum from declared "
+                f"{ImzMLAccessions.PROFILE_SPECTRUM} in fileDescription"
+            )
+            return SpectrumType.PROFILE
+
+        return None
+
+    def _guess_spectrum_type_from_storage_mode(self) -> Optional[str]:
+        """Last-resort guess: assume a processed-mode file is centroided.
+
+        Processed and centroid are orthogonal. ``IMS:1000031`` says each
+        spectrum carries its own m/z array; it says nothing about whether
+        the peaks in it are centroided or profile. Plenty of instruments
+        export profile spectra in processed mode -- ``bellini.imzML`` is
+        processed *and* declares ``MS:1000128 profile spectrum``.
+
+        This used to run before the declared accession was ever read, so
+        every processed file was reported as centroid whatever it said about
+        itself. It now runs only when the file declares neither
+        ``MS:1000127`` nor ``MS:1000128`` anywhere, where a guess is all
+        there is. Peak lists are the common case for processed exports, so
+        centroid is the better guess -- but it is still a guess, hence the
+        warning.
+        """
+        params = self._file_description_params()
+        if params is None:
             return None
 
-        params = file_desc.param_by_name
-        # If it's processed data, it's likely centroided
-        if params.get("processed", False):
-            logger.info("Assuming centroid spectrum for processed ImzML data")
+        if params.get(BinaryDataType.PROCESSED, False):
+            logger.warning(
+                "This imzML declares neither %s (centroid) nor %s (profile). "
+                "Assuming centroid, because it is stored in processed mode -- "
+                "but the two are independent, so check the result if the "
+                "spectra are actually profile.",
+                ImzMLAccessions.CENTROID_SPECTRUM,
+                ImzMLAccessions.PROFILE_SPECTRUM,
+            )
             return SpectrumType.CENTROID
 
         return None

thyra-2.2.2/thyra/converters/spatialdata/_chunking.py

@@ -1,41 +0,0 @@
-"""Image chunk/shard policy for Thyra-written SpatialData rasters.
-
-Single source of truth for how Thyra chunks the raster images it writes to
-SpatialData zarr -- and the designated seam for tile-aligned zarr v3 sharding
-once scverse/spatialdata PR #1106 (``raster_write_kwargs``) lands.
-
-Cross-repo contract: the viewer (Ousia) morphology tile server streams
-``DEFAULT_TILE_SIZE`` (= 512) edge tiles. Once sharding lands, the zarr INNER
-chunk edge must equal that so one served tile == one decompress. Thyra is a
-separate package and CANNOT import Ousia, so the coupling is DUPLICATED BY
-CONTRACT here -- keep ``INNER_TILE_EDGE`` numerically in sync with Ousia's
-``DEFAULT_TILE_SIZE``; a silent divergence reintroduces cold-tile amplification
-on the viewer. See the Ousia ADR docs/ADR-pyramid-tile-sharding.md.
-"""
-
-from typing import Tuple
-
-# Outer chunk edge used TODAY (pre-sharding the chunk IS the outer block): a 4096
-# block means a viewer 512-tile read decompresses at most one chunk per request.
-IMAGE_CHUNK_EDGE = 4096
-
-# Future zarr INNER chunk edge once #1106 lands == Ousia DEFAULT_TILE_SIZE (512).
-# Duplicated by contract (Thyra cannot import Ousia).
-INNER_TILE_EDGE = 512
-
-
-def image_chunks(spatial_ndim: int, edge: int = IMAGE_CHUNK_EDGE) -> Tuple[int, ...]:
-    """The zarr chunk tuple for a Thyra-written raster image (leading channel axis).
-
-    ``spatial_ndim == 2`` (c, y, x) -> ``(1, edge, edge)``;
-    ``spatial_ndim == 3`` (c, z, y, x) -> ``(1, 1, edge, edge)``.
-
-    This is the ONE place the future inner=``INNER_TILE_EDGE`` + outer-shard policy
-    will live when spatialdata PR #1106 ships ``raster_write_kwargs``. Today it
-    returns the pre-sharding behaviour, byte-for-byte.
-    """
-    if spatial_ndim == 2:
-        return (1, edge, edge)
-    if spatial_ndim == 3:
-        return (1, 1, edge, edge)
-    raise ValueError(f"spatial_ndim must be 2 or 3, got {spatial_ndim}")