thyra 2.0.1__tar.gz → 2.0.3__tar.gz

{thyra-2.0.1 → thyra-2.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thyra
-Version: 2.0.1
+Version: 2.0.3
 Summary: A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics
 License: MIT
 License-File: LICENSE

{thyra-2.0.1 → thyra-2.0.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "thyra"
-version = "2.0.1"
+version = "2.0.3"
 description = "A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics"
 authors = ["Theodoros Visvikis <t.visvikis@maastrichtuniversity.nl>"]
 maintainers = ["Theodoros Visvikis <t.visvikis@maastrichtuniversity.nl>"]

{thyra-2.0.1 → thyra-2.0.3}/thyra/__init__.py

@@ -36,7 +36,7 @@ warnings.filterwarnings(
     category=FutureWarning,
 )
 
-__version__ = "2.0.1"
+__version__ = "2.0.3"
 
 # Import key components - avoid wildcard imports
 try:

{thyra-2.0.1 → thyra-2.0.3}/thyra/converters/spatialdata/streaming_converter.py

@@ -40,6 +40,12 @@ if SPATIALDATA_AVAILABLE:
 
 logger = logging.getLogger(__name__)
 
+# Elements per chunk when building the string index arrays. Bounds the
+# transient cost of formatting to this many entries rather than the whole
+# axis; measured at 17.5 bytes per entry at peak against 88 for a one-shot
+# build, and 32 for an unchunked vectorised one.
+_INDEX_BUILD_CHUNK = 1_000_000
+
 
 class StreamingSpatialDataConverter(BaseSpatialDataConverter):
     """Memory-efficient streaming converter for MSI data to SpatialData format.
@@ -123,10 +129,21 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
         n_x, n_y, n_z = metadata.dimensions
         n_pixels = n_x * n_y * n_z
 
-        # Estimate number of m/z bins after resampling using the same
-        # resolution path the axis builder will use, so the gate against
-        # PCS_SIZE_THRESHOLD_GB sees the real bin count.
-        if self._resampling_config:
+        # Prefer the axis that was actually built. ``convert()`` runs
+        # ``_initialize_conversion()`` -- and so ``_setup_mass_axis()`` --
+        # before it asks which method to use, so by the time this gate is
+        # evaluated the real bin count is known and there is nothing to
+        # estimate. That matters most on the raw-axis path, where the
+        # fallback below assumes a 10 mDa spacing that the data need not
+        # have: a continuous file carrying 4,000 points over 250-1200 m/z
+        # was scored as though it had 95,000, and a processed file whose
+        # spectra share no m/z values was scored far too low, which routed
+        # the largest datasets to the method that holds the most in memory.
+        if self._common_mass_axis is not None:
+            n_mz_bins = len(self._common_mass_axis)
+        elif self._resampling_config:
+            # Same resolution path the axis builder will use, so the gate
+            # against PCS_SIZE_THRESHOLD_GB sees the real bin count.
             _, _, _, n_mz_bins = self._resolve_resampling_plan()
         else:
             min_mass, max_mass = metadata.mass_range
@@ -491,11 +508,19 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
         indptr_arr.attrs["encoding-type"] = "array"
         indptr_arr.attrs["encoding-version"] = "0.2.0"
 
+        # Column indices are bounded by n_cols, not by total_nnz, so they need
+        # their own dtype decision. Hardcoding int32 here wrapped silently
+        # above 2,147,483,647 m/z bins: zarr truncates an oversized write
+        # without warning, and the resulting negative indices survive all the
+        # way into scipy without an error. This is the same switch point scipy
+        # uses, so the matrix rebuilt from these arrays needs no cast.
+        indices_dtype = np.int64 if n_cols > np.iinfo(np.int32).max else np.int32
+
         chunk_size_zarr = min(total_nnz, 1000000)
         indices_arr = X_group.create_array(
             "indices",
             shape=(total_nnz,),
-            dtype=np.int32,
+            dtype=indices_dtype,
             chunks=(chunk_size_zarr,),
         )
         indices_arr.attrs["encoding-type"] = "array"
@@ -566,7 +591,11 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
                     pos = write_pos[pixel_idx]
                     positions = np.arange(pos, pos + nnz)
                     buf_positions.append(positions)
-                    buf_indices.append(mz_indices.astype(np.int32))
+                    # Take the dtype from the destination array rather than
+                    # re-deriving it, so the buffer and the store cannot drift
+                    # apart. Zarr accepts an oversized write and truncates it
+                    # silently, so a mismatch here would be invisible.
+                    buf_indices.append(mz_indices.astype(indices_arr.dtype))
                     buf_data.append(resampled_ints.astype(np.float64))
                     write_pos[pixel_idx] += nnz
                     buf_size += nnz
@@ -716,11 +745,15 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
 
         logger.info(f"Loaded CSR components: {len(data):,} entries")
 
-        # For large datasets (>2.1B entries), ensure 64-bit indices for scipy
+        # indices already carries the dtype chosen at write time, keyed on the
+        # column count (see _coo_setup_zarr_arrays), so there is nothing to fix
+        # up here. Upcasting it was never a safeguard anyway: it widened values
+        # that had already been truncated on the way in, and it was keyed on
+        # the wrong quantity. indptr is bounded by nnz, hence the check below.
+        # copy=False makes the no-op case free rather than a full-size copy.
         if len(data) > np.iinfo(np.int32).max:
             logger.info("Large dataset detected, using 64-bit sparse matrix indices")
-            indptr = indptr.astype(np.int64)
-            indices = indices.astype(np.int64)
+            indptr = indptr.astype(np.int64, copy=False)
 
         # Create CSR matrix directly (no COO intermediate)
         sparse_matrix: Union[sparse.csr_matrix, sparse.csc_matrix] = sparse.csr_matrix(
@@ -1417,7 +1450,9 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
 
         y_values = np.repeat(np.arange(n_y, dtype=np.int32), n_x)
         x_values = np.tile(np.arange(n_x, dtype=np.int32), n_y)
-        instance_ids = np.array([str(i) for i in range(n_rows)], dtype=str_dtype)
+        # Same reasoning as the var index below, but n_rows is the pixel count
+        # and stays far smaller than n_cols, so one shot needs no chunking.
+        instance_ids = np.arange(n_rows, dtype=np.int64).astype(str_dtype)
         spatial_x = x_values.astype(np.float64) * self.pixel_size_um
         spatial_y = y_values.astype(np.float64) * self.pixel_size_um
 
@@ -1464,7 +1499,17 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
         mz_values = self._common_mass_axis
         if mz_values is None:
             raise RuntimeError("Common mass axis not initialized")
-        mz_index = np.array([f"mz_{i}" for i in range(n_cols)], dtype=str_dtype)
+        # Built in chunks rather than from a list comprehension. Materialising
+        # n_cols Python str objects first costs about 88 bytes per entry at
+        # peak, against 17.5 for this; at 10 million bins that is 883 MB
+        # versus 175 MB, on a path whose docstring promises roughly 200 MB
+        # regardless of dataset size. The values are identical.
+        mz_index = np.empty(n_cols, dtype=str_dtype)
+        for start in range(0, n_cols, _INDEX_BUILD_CHUNK):
+            stop = min(start + _INDEX_BUILD_CHUNK, n_cols)
+            mz_index[start:stop] = np.strings.add(
+                "mz_", np.arange(start, stop, dtype=np.int64).astype(str_dtype)
+            )
         a = var_group.create_array("_index", data=mz_index)
         a.attrs["encoding-type"] = "string-array"
         a.attrs["encoding-version"] = "0.2.0"