thyra 3.7.0__tar.gz → 3.7.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. {thyra-3.7.0 → thyra-3.7.2}/PKG-INFO +1 -1
  2. {thyra-3.7.0 → thyra-3.7.2}/pyproject.toml +1 -1
  3. {thyra-3.7.0 → thyra-3.7.2}/thyra/__init__.py +1 -1
  4. {thyra-3.7.0 → thyra-3.7.2}/thyra/converters/spatialdata/base_spatialdata_converter.py +248 -42
  5. {thyra-3.7.0 → thyra-3.7.2}/thyra/converters/spatialdata/spatialdata_2d_converter.py +18 -8
  6. {thyra-3.7.0 → thyra-3.7.2}/thyra/converters/spatialdata/spatialdata_3d_converter.py +11 -6
  7. {thyra-3.7.0 → thyra-3.7.2}/thyra/converters/spatialdata/streaming_converter.py +33 -34
  8. {thyra-3.7.0 → thyra-3.7.2}/thyra/core/base_converter.py +60 -9
  9. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/extractors/imzml_extractor.py +8 -8
  10. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/imzml/imzml_reader.py +110 -31
  11. thyra-3.7.2/thyra/utils/pyimzml_direct.py +72 -0
  12. {thyra-3.7.0 → thyra-3.7.2}/.gitignore +0 -0
  13. {thyra-3.7.0 → thyra-3.7.2}/LICENSE +0 -0
  14. {thyra-3.7.0 → thyra-3.7.2}/README.md +0 -0
  15. {thyra-3.7.0 → thyra-3.7.2}/thyra/__main__.py +0 -0
  16. {thyra-3.7.0 → thyra-3.7.2}/thyra/alignment/__init__.py +0 -0
  17. {thyra-3.7.0 → thyra-3.7.2}/thyra/alignment/affine.py +0 -0
  18. {thyra-3.7.0 → thyra-3.7.2}/thyra/alignment/teaching_points.py +0 -0
  19. {thyra-3.7.0 → thyra-3.7.2}/thyra/config.py +0 -0
  20. {thyra-3.7.0 → thyra-3.7.2}/thyra/convert.py +0 -0
  21. {thyra-3.7.0 → thyra-3.7.2}/thyra/converters/__init__.py +0 -0
  22. {thyra-3.7.0 → thyra-3.7.2}/thyra/converters/spatialdata/__init__.py +0 -0
  23. {thyra-3.7.0 → thyra-3.7.2}/thyra/converters/spatialdata/_chunking.py +0 -0
  24. {thyra-3.7.0 → thyra-3.7.2}/thyra/converters/spatialdata/converter.py +0 -0
  25. {thyra-3.7.0 → thyra-3.7.2}/thyra/core/__init__.py +0 -0
  26. {thyra-3.7.0 → thyra-3.7.2}/thyra/core/base_extractor.py +0 -0
  27. {thyra-3.7.0 → thyra-3.7.2}/thyra/core/base_reader.py +0 -0
  28. {thyra-3.7.0 → thyra-3.7.2}/thyra/core/registry.py +0 -0
  29. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/__init__.py +0 -0
  30. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/extractors/__init__.py +0 -0
  31. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/extractors/bruker_extractor.py +0 -0
  32. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/extractors/phi_extractor.py +0 -0
  33. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/extractors/waters_extractor.py +0 -0
  34. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/ontology/__init__.py +0 -0
  35. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/ontology/_ims.py +0 -0
  36. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/ontology/_ms.py +0 -0
  37. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/ontology/_uo.py +0 -0
  38. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/ontology/cache.py +0 -0
  39. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/schema/__init__.py +0 -0
  40. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/schema/builder.py +0 -0
  41. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/schema/cli.py +0 -0
  42. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/schema/generate.py +0 -0
  43. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/schema/metaspace.py +0 -0
  44. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/schema/models.py +0 -0
  45. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/schema/msi_metadata.linkml.yaml +0 -0
  46. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/schema/msi_metadata_schema_v0_1.json +0 -0
  47. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/schema/store_io.py +0 -0
  48. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/schema/validate.py +0 -0
  49. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/schema/vocab.py +0 -0
  50. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/types.py +0 -0
  51. {thyra-3.7.0 → thyra-3.7.2}/thyra/metadata/validator.py +0 -0
  52. {thyra-3.7.0 → thyra-3.7.2}/thyra/preview.py +0 -0
  53. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/__init__.py +0 -0
  54. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/__init__.py +0 -0
  55. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/base_bruker_reader.py +0 -0
  56. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/folder_structure.py +0 -0
  57. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/mis_parser.py +0 -0
  58. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/rapiflex/__init__.py +0 -0
  59. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/rapiflex/rapiflex_reader.py +0 -0
  60. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/__init__.py +0 -0
  61. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/sdk/__init__.py +0 -0
  62. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/sdk/dll/LICENCE-BRUKER.txt +0 -0
  63. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/sdk/dll/README.md +0 -0
  64. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/sdk/dll/timsdata.dll +0 -0
  65. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/sdk/dll/timsdata.so +0 -0
  66. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/sdk/dll_manager.py +0 -0
  67. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/sdk/platform_detector.py +0 -0
  68. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/sdk/sdk_functions.py +0 -0
  69. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/timstof_reader.py +0 -0
  70. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/utils/__init__.py +0 -0
  71. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/utils/batch_processor.py +0 -0
  72. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/utils/coordinate_cache.py +0 -0
  73. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/utils/mass_axis_builder.py +0 -0
  74. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/bruker/timstof/utils/memory_manager.py +0 -0
  75. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/imzml/__init__.py +0 -0
  76. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/phi/__init__.py +0 -0
  77. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/phi/event_stream.py +0 -0
  78. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/phi/mass_axis.py +0 -0
  79. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/phi/phi_header.py +0 -0
  80. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/phi/phi_reader.py +0 -0
  81. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/waters/__init__.py +0 -0
  82. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/waters/imaging_grid.py +0 -0
  83. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/waters/lib/MLReader.dll +0 -0
  84. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/waters/lib/MassLynxRaw.dll +0 -0
  85. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/waters/lib/libMLReader.so +0 -0
  86. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/waters/lib/libMassLynxRaw.so +0 -0
  87. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/waters/masslynx_lib.py +0 -0
  88. {thyra-3.7.0 → thyra-3.7.2}/thyra/readers/waters/waters_reader.py +0 -0
  89. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/__init__.py +0 -0
  90. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/common_axis.py +0 -0
  91. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/constants.py +0 -0
  92. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/data_characteristics.py +0 -0
  93. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/decision_tree.py +0 -0
  94. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/gaps.py +0 -0
  95. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/instrument_detectors.py +0 -0
  96. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/mass_axis/__init__.py +0 -0
  97. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/mass_axis/base_generator.py +0 -0
  98. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/mass_axis/fticr_generator.py +0 -0
  99. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/mass_axis/linear_generator.py +0 -0
  100. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/mass_axis/linear_tof_generator.py +0 -0
  101. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/mass_axis/orbitrap_generator.py +0 -0
  102. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/mass_axis/reflector_tof_generator.py +0 -0
  103. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/strategies/__init__.py +0 -0
  104. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/strategies/base.py +0 -0
  105. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/strategies/nearest_neighbor.py +0 -0
  106. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/strategies/tic_preserving.py +0 -0
  107. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/tic.py +0 -0
  108. {thyra-3.7.0 → thyra-3.7.2}/thyra/resampling/types.py +0 -0
  109. {thyra-3.7.0 → thyra-3.7.2}/thyra/tools/__init__.py +0 -0
  110. {thyra-3.7.0 → thyra-3.7.2}/thyra/tools/check_ontology.py +0 -0
  111. {thyra-3.7.0 → thyra-3.7.2}/thyra/tools/make_example_data.py +0 -0
  112. {thyra-3.7.0 → thyra-3.7.2}/thyra/utils/__init__.py +0 -0
  113. {thyra-3.7.0 → thyra-3.7.2}/thyra/utils/bruker_exceptions.py +0 -0
  114. {thyra-3.7.0 → thyra-3.7.2}/thyra/utils/logging_config.py +0 -0
  115. {thyra-3.7.0 → thyra-3.7.2}/thyra/utils/windows_paths.py +0 -0
  116. {thyra-3.7.0 → thyra-3.7.2}/thyra/utils/zarr_atomic_write.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: thyra
3
- Version: 3.7.0
3
+ Version: 3.7.2
4
4
  Summary: A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics
5
5
  Project-URL: Homepage, https://github.com/M4i-Imaging-Mass-Spectrometry/thyra
6
6
  Project-URL: Repository, https://github.com/M4i-Imaging-Mass-Spectrometry/thyra
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "thyra"
7
- version = "3.7.0"
7
+ version = "3.7.2"
8
8
  description = "A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics"
9
9
  authors = [{ name = "Theodoros Visvikis", email = "t.visvikis@maastrichtuniversity.nl" }]
10
10
  maintainers = [{ name = "Theodoros Visvikis", email = "t.visvikis@maastrichtuniversity.nl" }]
@@ -36,7 +36,7 @@ warnings.filterwarnings(
36
36
  category=FutureWarning,
37
37
  )
38
38
 
39
- __version__ = "3.7.0"
39
+ __version__ = "3.7.2"
40
40
 
41
41
  # Import key components - avoid wildcard imports
42
42
  try:
@@ -215,6 +215,126 @@ def _calc_optical_scale_factors(
215
215
  return factors
216
216
 
217
217
 
218
+ def _nn_map_to_bins(
219
+ axis: NDArray[np.float64], mzs: NDArray[np.float64]
220
+ ) -> NDArray[np.int_]:
221
+ """Map in-range m/z values to their nearest bin on ``axis``.
222
+
223
+ Ties (a peak exactly between two bins) resolve to the right bin,
224
+ matching the strict ``<`` comparison this code has always used.
225
+
226
+ A module-level function rather than a method so the unbound-call test
227
+ harnesses (a ``SimpleNamespace`` posing as the converter) keep working.
228
+
229
+ Args:
230
+ axis: The target mass axis, ascending.
231
+ mzs: m/z values, all within ``[axis[0], axis[-1]]``.
232
+
233
+ Returns:
234
+ The nearest-bin index of each m/z value, same length as ``mzs``.
235
+ """
236
+ # Find insertion points using vectorized binary search
237
+ indices = np.searchsorted(axis, mzs)
238
+
239
+ # Clip to valid range. Everything reaching here is inside the axis,
240
+ # so this only pins searchsorted's one-past-the-end result for a
241
+ # value equal to axis[-1]; it can no longer pull an outside peak in.
242
+ indices_clipped = np.clip(indices, 0, len(axis) - 1)
243
+
244
+ # For non-boundary points, check if left is closer
245
+ # Only check where we're not at the left edge
246
+ check_left = indices > 0
247
+ if np.any(check_left):
248
+ # Get distances only for points that need checking
249
+ mz_values = axis[indices_clipped[check_left]]
250
+ mz_values_left = axis[indices_clipped[check_left] - 1]
251
+ mz_query = mzs[check_left]
252
+
253
+ # Use left if it's closer
254
+ use_left = np.abs(mz_values_left - mz_query) < np.abs(mz_values - mz_query)
255
+ indices_clipped[check_left] = np.where(
256
+ use_left, indices_clipped[check_left] - 1, indices_clipped[check_left]
257
+ )
258
+ return indices_clipped
259
+
260
+
261
+ def _nn_accumulate(
262
+ idx: NDArray[np.int_], intensities: NDArray[np.float64]
263
+ ) -> Tuple[NDArray[np.int_], NDArray[np.float64]]:
264
+ """Sum intensities per bin and return the non-zero bins, ascending.
265
+
266
+ Two equivalent routes. When ``idx`` is non-decreasing -- true whenever
267
+ the spectrum's m/z values are ascending, which every format seen so
268
+ far produces -- equal bins form contiguous runs, so the per-bin sums
269
+ are ``np.add.reduceat`` over the run starts. That is O(n_peaks),
270
+ where the ``np.bincount`` fallback is O(n_bins): for a centroid
271
+ spectrum of hundreds of peaks against an axis of 10^5 bins the
272
+ difference is roughly 7x per spectrum. Both sum left-to-right over
273
+ the same float64 values, so the results are bit-identical; the
274
+ fallback keeps unsorted input correct.
275
+
276
+ The kept-bin test is ``!= 0`` rather than ``> 0`` because a bin whose
277
+ accumulated value is negative is still a measurement: dropping it
278
+ silently raises the stored TIC above the input's. Baseline-subtracted
279
+ data can carry negative intensities, though every export seen so far
280
+ filters them out upstream.
281
+ """
282
+ # bincount always promotes its weights to float64; match it so both
283
+ # accumulation routes return the same dtype and the same rounding.
284
+ vals = intensities.astype(np.float64, copy=False)
285
+
286
+ if idx.size and bool(np.all(idx[1:] >= idx[:-1])):
287
+ starts = np.concatenate(([0], np.flatnonzero(np.diff(idx)) + 1))
288
+ if starts.size == idx.size:
289
+ # Every peak already sits in its own bin (the common case
290
+ # for both centroid and profile data): nothing to sum.
291
+ bins = idx
292
+ sums = vals
293
+ else:
294
+ bins = idx[starts]
295
+ sums = np.add.reduceat(vals, starts)
296
+ keep = sums != 0
297
+ return bins[keep].astype(np.int_, copy=False), sums[keep]
298
+
299
+ accumulated = np.bincount(idx, weights=vals)
300
+ nonzero_mask = accumulated != 0
301
+ nonzero_indices = np.where(nonzero_mask)[0].astype(np.int_)
302
+ nonzero_values = accumulated[nonzero_mask]
303
+ return nonzero_indices, nonzero_values.astype(np.float64)
304
+
305
+
306
+ class _SharedAxisNNCache:
307
+ """Precomputed nearest-neighbor mapping for one recurring m/z array.
308
+
309
+ Built the first time :meth:`_nearest_neighbor_resample` sees a spectrum,
310
+ and reused for every later spectrum that carries the same m/z array.
311
+ ``key`` is a private copy so a caller-side mutation of the original
312
+ array cannot fool the equality check; ``key_ref`` keeps the original
313
+ object for the O(1) identity test that readers yielding one shared
314
+ array (Rapiflex, Waters, PHI) hit every time.
315
+ """
316
+
317
+ __slots__ = (
318
+ "key",
319
+ "key_ref",
320
+ "n_total",
321
+ "n_dropped",
322
+ "lo",
323
+ "hi",
324
+ "starts",
325
+ "bins",
326
+ )
327
+
328
+ key: NDArray[np.float64]
329
+ key_ref: NDArray[np.float64]
330
+ n_total: int
331
+ n_dropped: int
332
+ lo: int
333
+ hi: int
334
+ starts: Optional[NDArray[np.int_]]
335
+ bins: NDArray[np.int_]
336
+
337
+
218
338
  class BaseSpatialDataConverter(BaseMSIConverter, ABC):
219
339
  """Base converter for MSI data to SpatialData format with shared functionality."""
220
340
 
@@ -345,6 +465,15 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
345
465
  # Cache for dense mass axis indices (to avoid repeated np.arange calls)
346
466
  self._cached_mass_axis_indices: Optional[NDArray[np.int_]] = None
347
467
 
468
+ # Shared-axis nearest-neighbor cache. Continuous imzML, Rapiflex,
469
+ # Waters and PHI all hand every spectrum the same m/z array, so the
470
+ # peak-to-bin mapping is computed once and verified per spectrum by
471
+ # array equality instead of being re-derived by searchsorted. None
472
+ # means "not built yet"; False means "tried and the data does not
473
+ # share an axis, stop checking". See _nearest_neighbor_resample.
474
+ self._nn_shared_cache: Any = None
475
+ self._nn_cache_misses: int = 0
476
+
348
477
  # Optical-MSI alignment (computed from FlexImaging Area definitions)
349
478
  self._alignment_result: Optional[AreaAlignmentResult] = None
350
479
  # Affine matrix mapping TIC raster indices to optical image pixels
@@ -1263,6 +1392,22 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
1263
1392
  raise ValueError("Common mass axis is not initialized")
1264
1393
 
1265
1394
  axis = self._common_mass_axis
1395
+
1396
+ # Shared-axis fast path: when every spectrum carries the same m/z
1397
+ # array (continuous imzML, Rapiflex, Waters, PHI), the peak-to-bin
1398
+ # mapping is a property of the axis pair, not of the spectrum. It is
1399
+ # computed once; each later spectrum only proves its m/z array is
1400
+ # the same one -- an exact array comparison, which is far cheaper
1401
+ # than re-deriving the mapping by binary search per spectrum.
1402
+ # ``False`` means the cache disabled itself (processed-mode data).
1403
+ # getattr rather than a bare read: test harnesses drive this method
1404
+ # unbound on a stub that predates the cache, and they exercise the
1405
+ # generic path below, which is exactly what "no cache" selects.
1406
+ if getattr(self, "_nn_shared_cache", False) is not False:
1407
+ result = self._nn_resample_via_cache(axis, mzs, intensities)
1408
+ if result is not None:
1409
+ return result
1410
+
1266
1411
  in_range = (mzs >= axis[0]) & (mzs <= axis[-1])
1267
1412
  if not in_range.all():
1268
1413
  self._count_out_of_range(int(mzs.size - in_range.sum()), int(mzs.size))
@@ -1271,45 +1416,101 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
1271
1416
  if mzs.size == 0:
1272
1417
  return np.array([], dtype=np.int_), np.array([], dtype=np.float64)
1273
1418
 
1274
- # Find insertion points using vectorized binary search
1275
- indices = np.searchsorted(axis, mzs)
1276
-
1277
- # OPTIMIZED: Handle boundary and nearest neighbor in one pass
1278
- # Clip to valid range. Everything reaching here is inside the axis,
1279
- # so this only pins searchsorted's one-past-the-end result for a
1280
- # value equal to axis[-1]; it can no longer pull an outside peak in.
1281
- indices_clipped = np.clip(indices, 0, len(axis) - 1)
1282
-
1283
- # For non-boundary points, check if left is closer
1284
- # Only check where we're not at the left edge
1285
- check_left = indices > 0
1286
- if np.any(check_left):
1287
- # Get distances only for points that need checking
1288
- mz_values = axis[indices_clipped[check_left]]
1289
- mz_values_left = axis[indices_clipped[check_left] - 1]
1290
- mz_query = mzs[check_left]
1291
-
1292
- # Use left if it's closer
1293
- use_left = np.abs(mz_values_left - mz_query) < np.abs(mz_values - mz_query)
1294
- indices_clipped[check_left] = np.where(
1295
- use_left, indices_clipped[check_left] - 1, indices_clipped[check_left]
1296
- )
1419
+ return _nn_accumulate(_nn_map_to_bins(axis, mzs), intensities)
1297
1420
 
1298
- # OPTIMIZATION: Use pandas-style groupby or bincount for accumulation
1299
- # bincount without minlength is fastest
1300
- accumulated = np.bincount(indices_clipped, weights=intensities)
1421
+ def _build_nn_shared_cache(
1422
+ self, axis: NDArray[np.float64], mzs: NDArray[np.float64]
1423
+ ) -> Optional[_SharedAxisNNCache]:
1424
+ """Precompute the nearest-neighbor mapping for one m/z array.
1301
1425
 
1302
- # Extract non-zero bins for sparse representation. The test is
1303
- # ``!= 0`` rather than ``> 0`` because a bin whose accumulated value is
1304
- # negative is still a measurement: dropping it silently raises the
1305
- # stored TIC above the input's. Baseline-subtracted data can carry
1306
- # negative intensities, though every export seen so far filters them
1307
- # out upstream.
1308
- nonzero_mask = accumulated != 0
1309
- nonzero_indices = np.where(nonzero_mask)[0].astype(np.int_)
1310
- nonzero_values = accumulated[nonzero_mask]
1426
+ Returns None when the array cannot be cached -- empty, or not
1427
+ ascending, which the contiguous in-range slice below relies on.
1428
+ The mapping itself comes from the same :meth:`_nn_map_to_bins`
1429
+ the generic path uses, so a cache hit and a fresh computation
1430
+ agree bin for bin.
1431
+ """
1432
+ if mzs.size == 0 or not bool(np.all(mzs[1:] >= mzs[:-1])):
1433
+ return None
1434
+
1435
+ # Ascending m/z makes the in-range subset one contiguous slice,
1436
+ # with the same inclusive endpoints as the generic path's mask.
1437
+ lo = int(np.searchsorted(mzs, axis[0], side="left"))
1438
+ hi = int(np.searchsorted(mzs, axis[-1], side="right"))
1439
+
1440
+ cache = _SharedAxisNNCache()
1441
+ cache.key = mzs.copy()
1442
+ cache.key_ref = mzs
1443
+ cache.n_total = int(mzs.size)
1444
+ cache.n_dropped = int(mzs.size - (hi - lo))
1445
+ cache.lo = lo
1446
+ cache.hi = hi
1447
+ if hi <= lo:
1448
+ cache.starts = None
1449
+ cache.bins = np.array([], dtype=np.int_)
1450
+ return cache
1451
+
1452
+ idx = _nn_map_to_bins(axis, mzs[lo:hi])
1453
+ # Ascending m/z onto an ascending axis gives non-decreasing bins,
1454
+ # so equal bins form contiguous runs.
1455
+ starts = np.concatenate(([0], np.flatnonzero(np.diff(idx)) + 1))
1456
+ if starts.size == idx.size:
1457
+ # Every peak in its own bin: per-spectrum work reduces to a
1458
+ # zero-filter over the raw intensities.
1459
+ cache.starts = None
1460
+ cache.bins = idx.astype(np.int_, copy=False)
1461
+ else:
1462
+ cache.starts = starts
1463
+ cache.bins = idx[starts].astype(np.int_, copy=False)
1464
+ return cache
1465
+
1466
+ def _nn_resample_via_cache(
1467
+ self,
1468
+ axis: NDArray[np.float64],
1469
+ mzs: NDArray[np.float64],
1470
+ intensities: NDArray[np.float64],
1471
+ ) -> Optional[Tuple[NDArray[np.int_], NDArray[np.float64]]]:
1472
+ """Resample through the shared-axis cache, or return None to decline.
1473
+
1474
+ The cache is built from the first spectrum seen. A hit requires the
1475
+ spectrum's m/z array to be the cached one -- same object, or equal
1476
+ element for element -- so a lying reader cannot get a stale mapping;
1477
+ it can only miss. After five consecutive misses the cache disables
1478
+ itself so processed-mode data stops paying for the comparison
1479
+ (its size check is O(1) in the common case anyway).
1480
+ """
1481
+ cache = self._nn_shared_cache
1482
+ if cache is False:
1483
+ return None
1484
+ if cache is None:
1485
+ cache = self._build_nn_shared_cache(axis, mzs)
1486
+ if cache is None:
1487
+ self._nn_shared_cache = False
1488
+ return None
1489
+ self._nn_shared_cache = cache
1490
+
1491
+ if not (
1492
+ mzs is cache.key_ref
1493
+ or (mzs.size == cache.n_total and bool(np.array_equal(mzs, cache.key)))
1494
+ ):
1495
+ self._nn_cache_misses += 1
1496
+ if self._nn_cache_misses > 4:
1497
+ self._nn_shared_cache = False
1498
+ return None
1311
1499
 
1312
- return nonzero_indices, nonzero_values.astype(np.float64)
1500
+ if cache.n_dropped:
1501
+ self._count_out_of_range(cache.n_dropped, cache.n_total)
1502
+ if cache.hi <= cache.lo:
1503
+ return np.array([], dtype=np.int_), np.array([], dtype=np.float64)
1504
+
1505
+ # Match bincount's float64 promotion so cached and generic results
1506
+ # carry the same rounding.
1507
+ vals = intensities[cache.lo : cache.hi].astype(np.float64, copy=False)
1508
+ if cache.starts is None:
1509
+ sums = vals
1510
+ else:
1511
+ sums = np.add.reduceat(vals, cache.starts)
1512
+ keep = sums != 0
1513
+ return cache.bins[keep], sums[keep]
1313
1514
 
1314
1515
  def _tic_preserving_resample(
1315
1516
  self, mzs: NDArray[np.float64], intensities: NDArray[np.float64]
@@ -1798,6 +1999,8 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
1798
1999
  )
1799
2000
  else:
1800
2001
  # Standard physical coordinates (micrometers)
2002
+ from shapely import box as shapely_box_vectorized
2003
+
1801
2004
  x_coords: NDArray[np.float64] = adata.obs["spatial_x"].values
1802
2005
  y_coords: NDArray[np.float64] = adata.obs["spatial_y"].values
1803
2006
  half_pixel_um = self.pixel_size_um / 2
@@ -1805,12 +2008,15 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
1805
2008
  # Footprints are flat, on every route including volumes. A
1806
2009
  # slice's depth lives on the TIC image's Scale and in
1807
2010
  # obs["spatial_z"]; see the docstring for why it is not also
1808
- # put on the geometry.
1809
- for i in range(len(adata)):
1810
- x, y = x_coords[i], y_coords[i]
1811
- x0, y0 = x - half_pixel_um, y - half_pixel_um
1812
- x1, y1 = x + half_pixel_um, y + half_pixel_um
1813
- geometries.append(box(x0, y0, x1, y1))
2011
+ # put on the geometry. Built through shapely's vectorised box
2012
+ # constructor -- one C call for the whole table instead of one
2013
+ # Python-level geometry per pixel.
2014
+ geometries = shapely_box_vectorized(
2015
+ x_coords - half_pixel_um,
2016
+ y_coords - half_pixel_um,
2017
+ x_coords + half_pixel_um,
2018
+ y_coords + half_pixel_um,
2019
+ )
1814
2020
 
1815
2021
  # Create GeoDataFrame with appropriate index
1816
2022
  if valid_indices is not None:
@@ -206,20 +206,30 @@ class SpatialData2DConverter(BaseSpatialDataConverter):
206
206
  # Calculate TIC for this pixel
207
207
  tic_value = float(np.sum(intensities))
208
208
 
209
- # Update total intensity for average spectrum calculation using sparse indexing
210
- # OPTIMIZATION: Use np.add.at for vectorized sparse accumulation
211
- np.add.at(data_structures["total_intensity"], mz_indices, intensities)
209
+ # Update total intensity for average spectrum calculation. The dense
210
+ # resampling path hands the cached full-axis arange, where a plain
211
+ # vector add is ~5x cheaper than the scatter np.add.at performs;
212
+ # sparse indices keep the scatter.
213
+ total_intensity = data_structures["total_intensity"]
214
+ is_dense = (
215
+ mz_indices is self._cached_mass_axis_indices
216
+ or mz_indices is self._identity_mass_indices
217
+ ) and intensities.size == total_intensity.size
218
+ if is_dense:
219
+ total_intensity += intensities
220
+ else:
221
+ np.add.at(total_intensity, mz_indices, intensities)
212
222
  data_structures["pixel_count"] += 1
213
223
 
214
224
  # Per-region accumulation for multi-region datasets
215
225
  if "region_total_intensity" in data_structures:
216
226
  region_num = self._region_map.get((x, y), -1)
217
227
  if region_num in data_structures["region_total_intensity"]:
218
- np.add.at(
219
- data_structures["region_total_intensity"][region_num],
220
- mz_indices,
221
- intensities,
222
- )
228
+ region_total = data_structures["region_total_intensity"][region_num]
229
+ if is_dense:
230
+ region_total += intensities
231
+ else:
232
+ np.add.at(region_total, mz_indices, intensities)
223
233
  data_structures["region_pixel_count"][region_num] += 1
224
234
 
225
235
  # Add data to the appropriate slice
@@ -120,12 +120,17 @@ class SpatialData3DConverter(BaseSpatialDataConverter):
120
120
  # Resampled case - intensities match common mass axis length
121
121
  data_structures["total_intensity"] += intensities
122
122
  else:
123
- # Non-resampled case - need to map to indices
124
- for i, intensity in enumerate(intensities):
125
- if i < len(mz_indices) and mz_indices[i] < len(
126
- data_structures["total_intensity"]
127
- ):
128
- data_structures["total_intensity"][mz_indices[i]] += intensity
123
+ # Sparse case - scatter onto the mapped indices. Guard the
124
+ # lengths the way the old per-element loop did: only pairs
125
+ # that exist in both arrays, only in-bounds indices.
126
+ n = min(len(intensities), len(mz_indices))
127
+ idx = mz_indices[:n]
128
+ in_bounds = idx < len(data_structures["total_intensity"])
129
+ np.add.at(
130
+ data_structures["total_intensity"],
131
+ idx[in_bounds],
132
+ intensities[:n][in_bounds],
133
+ )
129
134
  data_structures["pixel_count"] += 1
130
135
 
131
136
  # Per-region accumulation for multi-region datasets
@@ -24,6 +24,7 @@ from numpy.typing import NDArray
24
24
  from scipy import sparse
25
25
  from tqdm import tqdm
26
26
 
27
+ from ...resampling import ResamplingMethod
27
28
  from .base_spatialdata_converter import (
28
29
  SPATIALDATA_AVAILABLE,
29
30
  BaseSpatialDataConverter,
@@ -125,6 +126,15 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
125
126
  self._zarr_store: Optional[zarr.Group] = None
126
127
  self._temp_path: Optional[Path] = None
127
128
 
129
+ # Resolved once here rather than per spectrum: _process_spectrum is
130
+ # the hottest call in both passes, and re-importing ResamplingMethod
131
+ # plus re-checking the attribute there measured ~35 us per call.
132
+ self._nn_route: bool = (
133
+ self._resampling_config is not None
134
+ and getattr(self, "_resampling_method", None)
135
+ == ResamplingMethod.NEAREST_NEIGHBOR
136
+ )
137
+
128
138
  def _suppress_reader_progress(self) -> None:
129
139
  """Suppress progress output from reader during streaming passes."""
130
140
  setattr(self.reader, "_quiet_mode", True)
@@ -750,19 +760,17 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
750
760
  all_idx = all_idx[order]
751
761
  all_dat = all_dat[order]
752
762
 
753
- # Write contiguous runs in bulk
763
+ # Find run boundaries vectorised, then write one Zarr slice per
764
+ # contiguous run. The previous element-at-a-time Python walk cost
765
+ # ~2.5 s per 5M-entry flush against ~27 ms for the same answer.
754
766
  n = len(all_pos)
755
- i = 0
756
- while i < n:
757
- start_pos = all_pos[i]
758
- # Find end of contiguous run
759
- j = i + 1
760
- while j < n and all_pos[j] == all_pos[j - 1] + 1:
761
- j += 1
762
- # Write the contiguous block
763
- indices_arr[int(start_pos) : int(start_pos) + (j - i)] = all_idx[i:j]
764
- data_arr[int(start_pos) : int(start_pos) + (j - i)] = all_dat[i:j]
765
- i = j
767
+ breaks = np.flatnonzero(np.diff(all_pos) != 1)
768
+ starts = np.concatenate(([0], breaks + 1))
769
+ ends = np.concatenate((breaks + 1, [n]))
770
+ for i, j in zip(starts.tolist(), ends.tolist()):
771
+ start_pos = int(all_pos[i])
772
+ indices_arr[start_pos : start_pos + (j - i)] = all_idx[i:j]
773
+ data_arr[start_pos : start_pos + (j - i)] = all_dat[i:j]
766
774
 
767
775
  del all_pos, all_idx, all_dat
768
776
  gc.collect()
@@ -787,21 +795,18 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
787
795
  mz_indices = self._map_mass_to_indices(mzs)
788
796
  return mz_indices, intensities
789
797
 
790
- # Check for optimized nearest-neighbor path
791
- if hasattr(self, "_resampling_method"):
792
- from ...resampling import ResamplingMethod
793
-
794
- if self._resampling_method == ResamplingMethod.NEAREST_NEIGHBOR:
795
- mz_indices, resampled = self._nearest_neighbor_resample(
796
- mzs, intensities
797
- )
798
- return mz_indices, resampled
798
+ # Optimized nearest-neighbor path; the route is resolved once in
799
+ # __init__ because this runs once per spectrum per pass.
800
+ if self._nn_route:
801
+ return self._nearest_neighbor_resample(mzs, intensities)
799
802
 
800
803
  # Fallback: general resampling with zero filtering
801
804
  resampled_ints = self._resample_spectrum(mzs, intensities)
802
805
  if self._common_mass_axis is None:
803
806
  raise RuntimeError("Common mass axis not initialized")
804
- mz_indices = np.arange(len(self._common_mass_axis))
807
+ mz_indices = self._cached_mass_axis_indices
808
+ if mz_indices is None or mz_indices.size != len(self._common_mass_axis):
809
+ mz_indices = np.arange(len(self._common_mass_axis))
805
810
  mask = resampled_ints != 0
806
811
  return mz_indices[mask], resampled_ints[mask]
807
812
 
@@ -1373,10 +1378,6 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
1373
1378
  # Current write position for each column
1374
1379
  write_pos = indptr[:-1].copy()
1375
1380
 
1376
- # Periodic flush to reduce memory pressure
1377
- flush_interval = 100_000
1378
- spectra_since_flush = 0
1379
-
1380
1381
  # Reset reader for second pass
1381
1382
  # For real readers (ImzML, Bruker), iter_spectra() is a generator factory
1382
1383
  # that creates a fresh iterator each time - no need to recreate the reader.
@@ -1419,15 +1420,13 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
1419
1420
  write_pos[mz_indices] += 1
1420
1421
 
1421
1422
  pbar.update(1)
1422
- spectra_since_flush += 1
1423
-
1424
- # Periodic flush
1425
- if spectra_since_flush >= flush_interval:
1426
- mm_indices.flush()
1427
- mm_data.flush()
1428
- spectra_since_flush = 0
1429
1423
 
1430
- # Final flush
1424
+ # One flush at the end. A periodic flush used to run every 100k
1425
+ # spectra, but ``np.memmap.flush`` writes back the whole mapping
1426
+ # synchronously, serialising disk writeback into the compute loop;
1427
+ # the OS pager already writes dirty pages back in the background.
1428
+ # Scatter writes each position exactly once, so nothing is dirtied
1429
+ # twice and the deferred writeback costs the same total I/O.
1431
1430
  mm_indices.flush()
1432
1431
  mm_data.flush()
1433
1432
 
@@ -110,6 +110,11 @@ class BaseMSIConverter(ABC):
110
110
  self.z_spacing_source = ZSpacingSource.ASSUMED_ISOTROPIC
111
111
  self.options: Dict[str, Any] = kwargs
112
112
  self._common_mass_axis: Optional[NDArray[np.float64]] = None
113
+ # Identity-mapping cache for _map_mass_to_indices: shared-axis
114
+ # readers hand every spectrum the very m/z array the common axis
115
+ # was built from, so the exact-match search is the identity map.
116
+ self._identity_mass_indices: Optional[NDArray[np.int_]] = None
117
+ self._axis_strictly_increasing: Optional[bool] = None
113
118
  self._dimensions: Optional[Tuple[int, int, int]] = None
114
119
  self._metadata: Optional[dict[str, Any]] = None
115
120
  from ..config import DEFAULT_BUFFER_SIZE
@@ -568,11 +573,23 @@ class BaseMSIConverter(ABC):
568
573
  def _map_mass_to_indices(self, mzs: NDArray[np.float64]) -> NDArray[np.int_]:
569
574
  """Map m/z values to indices in the common mass axis with high accuracy.
570
575
 
576
+ The returned array is parallel to ``mzs`` (same length, same order),
577
+ so callers may pair it element-for-element with the spectrum's
578
+ intensity array.
579
+
571
580
  Args:
572
581
  mzs: Array of m/z values
573
582
 
574
583
  Returns:
575
- Array of indices in common mass axis
584
+ Array of indices in common mass axis, parallel to ``mzs``
585
+
586
+ Raises:
587
+ ValueError: If any m/z value has no common-axis entry within
588
+ tolerance. Without resampling the axis is built from the
589
+ spectra themselves, so every value must match exactly; a
590
+ near-miss means the axis and the data have diverged, and
591
+ dropping the value silently would desync the index and
592
+ intensity arrays downstream.
576
593
  """
577
594
  if self._common_mass_axis is None:
578
595
  raise ValueError("Common mass axis is not initialized.")
@@ -580,18 +597,52 @@ class BaseMSIConverter(ABC):
580
597
  if mzs.size == 0:
581
598
  return np.array([], dtype=int)
582
599
 
583
- # Use searchsorted for exact mapping
584
- indices = np.searchsorted(self._common_mass_axis, mzs)
585
-
586
- # Ensure indices are within bounds
587
- indices = np.clip(indices, 0, len(self._common_mass_axis) - 1)
600
+ axis = self._common_mass_axis
601
+
602
+ # Identity fast path. On a shared-axis reader (continuous imzML,
603
+ # Rapiflex, Waters, PHI) every spectrum's m/z array is the very
604
+ # array the common axis was built from, so the exact-match search
605
+ # below returns 0..n-1 -- yet used to be re-derived by binary
606
+ # search per spectrum, twice per streaming conversion. Equality is
607
+ # proven, not assumed: an O(1) size check gates a full array
608
+ # comparison, and the arange shortcut is only valid when the axis
609
+ # is strictly increasing (searchsorted maps duplicates to their
610
+ # first occurrence, which is not the identity).
611
+ if mzs.size == axis.size and (mzs is axis or bool(np.array_equal(mzs, axis))):
612
+ if self._axis_strictly_increasing is None:
613
+ self._axis_strictly_increasing = bool(np.all(np.diff(axis) > 0))
614
+ if self._axis_strictly_increasing:
615
+ cached = self._identity_mass_indices
616
+ if cached is None or cached.size != axis.size:
617
+ cached = np.arange(axis.size, dtype=np.intp)
618
+ self._identity_mass_indices = cached
619
+ return cached
620
+
621
+ # searchsorted returns the right-hand neighbor; the nearest axis
622
+ # entry may sit on either side, so compare both before validating.
623
+ right = np.clip(np.searchsorted(axis, mzs), 0, len(axis) - 1)
624
+ left = np.maximum(right - 1, 0)
625
+ indices = np.where(
626
+ np.abs(axis[right] - mzs) <= np.abs(axis[left] - mzs), right, left
627
+ )
588
628
 
589
- # For complete accuracy, validate the indices
590
629
  # Very small tolerance threshold for floating point differences
591
630
  max_diff = 1e-6
592
- mask = np.abs(self._common_mass_axis[indices] - mzs) <= max_diff
631
+ diffs = np.abs(axis[indices] - mzs)
632
+ bad = diffs > max_diff
633
+ if np.any(bad):
634
+ worst = int(np.argmax(diffs))
635
+ raise ValueError(
636
+ f"{int(np.count_nonzero(bad))} of {mzs.size} m/z values have "
637
+ f"no common mass axis entry within {max_diff:g} (worst: m/z "
638
+ f"{mzs[worst]!r} is {diffs[worst]:.6g} from nearest axis "
639
+ f"entry {axis[indices[worst]]!r}). The common mass axis does "
640
+ f"not cover this spectrum; refusing to drop values silently "
641
+ f"because callers pair these indices with the full intensity "
642
+ f"array."
643
+ )
593
644
 
594
- return indices[mask]
645
+ return indices
595
646
 
596
647
  def _add_to_sparse_matrix(
597
648
  self,