thyra 2.2.3__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {thyra-2.2.3 → thyra-2.3.0}/PKG-INFO +2 -2
- {thyra-2.2.3 → thyra-2.3.0}/pyproject.toml +11 -2
- {thyra-2.2.3 → thyra-2.3.0}/thyra/__init__.py +1 -1
- {thyra-2.2.3 → thyra-2.3.0}/thyra/__main__.py +28 -4
- {thyra-2.2.3 → thyra-2.3.0}/thyra/convert.py +11 -0
- thyra-2.3.0/thyra/converters/spatialdata/_chunking.py +89 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/converters/spatialdata/base_spatialdata_converter.py +81 -49
- {thyra-2.2.3 → thyra-2.3.0}/thyra/converters/spatialdata/spatialdata_3d_converter.py +5 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/converters/spatialdata/streaming_converter.py +22 -34
- {thyra-2.2.3 → thyra-2.3.0}/thyra/metadata/extractors/imzml_extractor.py +75 -4
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/imzml/imzml_reader.py +17 -1
- thyra-2.3.0/thyra/resampling/constants.py +91 -0
- thyra-2.2.3/thyra/converters/spatialdata/_chunking.py +0 -41
- thyra-2.2.3/thyra/resampling/constants.py +0 -44
- {thyra-2.2.3 → thyra-2.3.0}/LICENSE +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/README.md +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/alignment/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/alignment/affine.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/alignment/teaching_points.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/config.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/converters/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/converters/spatialdata/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/converters/spatialdata/converter.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/converters/spatialdata/spatialdata_2d_converter.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/core/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/core/base_converter.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/core/base_extractor.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/core/base_reader.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/core/registry.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/metadata/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/metadata/extractors/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/metadata/extractors/bruker_extractor.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/metadata/extractors/waters_extractor.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/metadata/ontology/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/metadata/ontology/_ims.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/metadata/ontology/_ms.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/metadata/ontology/_uo.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/metadata/ontology/cache.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/metadata/types.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/metadata/validator.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/preview.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/base_bruker_reader.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/folder_structure.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/mis_parser.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/rapiflex/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/rapiflex/rapiflex_reader.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/sdk/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/sdk/dll/LICENCE-BRUKER.txt +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/sdk/dll/README.md +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/sdk/dll/timsdata.dll +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/sdk/dll/timsdata.so +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/sdk/dll_manager.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/sdk/platform_detector.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/sdk/sdk_functions.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/timstof_reader.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/utils/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/utils/batch_processor.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/utils/coordinate_cache.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/utils/mass_axis_builder.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/bruker/timstof/utils/memory_manager.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/imzml/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/waters/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/waters/imaging_grid.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/waters/lib/MLReader.dll +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/waters/lib/MassLynxRaw.dll +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/waters/lib/libMLReader.so +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/waters/lib/libMassLynxRaw.so +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/waters/masslynx_lib.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/readers/waters/waters_reader.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/common_axis.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/data_characteristics.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/decision_tree.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/gaps.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/instrument_detectors.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/mass_axis/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/mass_axis/base_generator.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/mass_axis/fticr_generator.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/mass_axis/linear_generator.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/mass_axis/linear_tof_generator.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/mass_axis/orbitrap_generator.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/mass_axis/reflector_tof_generator.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/strategies/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/strategies/base.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/strategies/nearest_neighbor.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/strategies/tic_preserving.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/tic.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/resampling/types.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/tools/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/tools/check_ontology.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/tools/make_example_data.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/utils/__init__.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/utils/bruker_exceptions.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/utils/logging_config.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/utils/windows_paths.py +0 -0
- {thyra-2.2.3 → thyra-2.3.0}/thyra/utils/zarr_atomic_write.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: thyra
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.0
|
|
4
4
|
Summary: A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -39,7 +39,7 @@ Requires-Dist: pyimzML (>=1.4.0)
|
|
|
39
39
|
Requires-Dist: scipy (>=1.7.0)
|
|
40
40
|
Requires-Dist: spatialdata (>=0.8.0,<0.9)
|
|
41
41
|
Requires-Dist: tqdm (>=4.50.0)
|
|
42
|
-
Requires-Dist: zarr (>=3.
|
|
42
|
+
Requires-Dist: zarr (>=3.1.4,<3.2)
|
|
43
43
|
Project-URL: Bug Tracker, https://github.com/M4i-Imaging-Mass-Spectrometry/thyra/issues
|
|
44
44
|
Project-URL: Changelog, https://github.com/M4i-Imaging-Mass-Spectrometry/thyra/blob/main/CHANGELOG.md
|
|
45
45
|
Project-URL: Contributing, https://github.com/M4i-Imaging-Mass-Spectrometry/thyra/blob/main/CONTRIBUTING.md
|
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "thyra"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.3.0"
|
|
8
8
|
description = "A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics"
|
|
9
9
|
authors = ["Theodoros Visvikis <t.visvikis@maastrichtuniversity.nl>"]
|
|
10
10
|
maintainers = ["Theodoros Visvikis <t.visvikis@maastrichtuniversity.nl>"]
|
|
@@ -86,7 +86,16 @@ spatialdata = ">=0.8.0, <0.9" # tested 0.8.0
|
|
|
86
86
|
# bit ome_zarr 0.12.x with newer dask, and spatialdata 0.8 requires >= 0.16.
|
|
87
87
|
ome-zarr = ">=0.16.0, <0.19" # tested 0.18.0
|
|
88
88
|
tqdm = ">=4.50.0"
|
|
89
|
-
|
|
89
|
+
# Floor is 3.1.4, and it is load-bearing. anndata >= 0.13 writes every zarr v3
|
|
90
|
+
# array with `shards="auto"`. zarr < 3.1.4 sized the auto chunk with
|
|
91
|
+
# `max_bytes=1024` where its own comment says "aim for a 1MiB chunk" (compare
|
|
92
|
+
# `_auto_partition` in 3.1.3 vs 3.1.4), so one shard -- one FILE -- came out at
|
|
93
|
+
# ~1 KB. Measured on test_data/pea.imzML: 393,310 files for a single table and
|
|
94
|
+
# 434 s of a 445 s conversion spent in the write, against 12 files and 2.7 s on
|
|
95
|
+
# 3.1.6. Nothing in Thyra changed; the anndata 0.13 bump alone did this.
|
|
96
|
+
# 3.1.4 is also where `array.target_shard_size_bytes` arrives, which is the
|
|
97
|
+
# knob _chunking.table_write_config sets to claim the budget from anndata.
|
|
98
|
+
zarr = ">=3.1.4, <3.2" # tested 3.1.6
|
|
90
99
|
imagecodecs = ">=2024.1.1" # For reading compressed TIFF optical images
|
|
91
100
|
# Floor is 0.13.2, not 0.13.0. anndata 0.13.0 imports
|
|
92
101
|
# `typing_extensions.sentinel` (needs typing-extensions >= 4.16) but declares
|
|
@@ -261,11 +261,21 @@ def _build_resampling_config(
|
|
|
261
261
|
def _build_reader_options(
|
|
262
262
|
use_recalibrated: bool,
|
|
263
263
|
intensity_threshold: Optional[float],
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
264
|
+
spectrum_type: str = "auto",
|
|
265
|
+
) -> dict[str, bool | float | str]:
|
|
266
|
+
"""Build reader options dictionary from CLI parameters.
|
|
267
|
+
|
|
268
|
+
``spectrum_type="auto"`` is the CLI's way of saying "no override", so it is
|
|
269
|
+
omitted entirely rather than forwarded -- readers take ``None``/absent to
|
|
270
|
+
mean detect, and ``"auto"`` is not a representation.
|
|
271
|
+
"""
|
|
272
|
+
options: dict[str, bool | float | str] = {
|
|
273
|
+
"use_recalibrated_state": use_recalibrated
|
|
274
|
+
}
|
|
267
275
|
if intensity_threshold is not None:
|
|
268
276
|
options["intensity_threshold"] = intensity_threshold
|
|
277
|
+
if spectrum_type != "auto":
|
|
278
|
+
options["spectrum_type"] = spectrum_type
|
|
269
279
|
return options
|
|
270
280
|
|
|
271
281
|
|
|
@@ -548,6 +558,17 @@ class GroupedCommand(click.Command):
|
|
|
548
558
|
"(default: no limit)"
|
|
549
559
|
),
|
|
550
560
|
)
|
|
561
|
+
@click.option(
|
|
562
|
+
"--spectrum-type",
|
|
563
|
+
type=click.Choice(["auto", "profile", "centroid"]),
|
|
564
|
+
default="auto",
|
|
565
|
+
help=(
|
|
566
|
+
"Spectrum representation (default: auto-detect from the file). "
|
|
567
|
+
"Overrides what the imzML declares via MS:1000127/MS:1000128; "
|
|
568
|
+
"contradicting a declaration is logged as a warning. SCiLS Lab spells "
|
|
569
|
+
"this --rep_type. imzML only."
|
|
570
|
+
),
|
|
571
|
+
)
|
|
551
572
|
# -- Bruker-specific --
|
|
552
573
|
@click.option(
|
|
553
574
|
"--use-recalibrated/--no-recalibrated",
|
|
@@ -597,6 +618,7 @@ def main(
|
|
|
597
618
|
resample_reference_mz: float,
|
|
598
619
|
resample_gap_tolerance: Optional[float],
|
|
599
620
|
mass_axis_type: str,
|
|
621
|
+
spectrum_type: str,
|
|
600
622
|
sparse_format: str,
|
|
601
623
|
include_optical: bool,
|
|
602
624
|
intensity_threshold: Optional[float],
|
|
@@ -664,7 +686,9 @@ def main(
|
|
|
664
686
|
)
|
|
665
687
|
|
|
666
688
|
# Build reader options for format-specific settings
|
|
667
|
-
reader_options = _build_reader_options(
|
|
689
|
+
reader_options = _build_reader_options(
|
|
690
|
+
use_recalibrated, intensity_threshold, spectrum_type
|
|
691
|
+
)
|
|
668
692
|
|
|
669
693
|
# Perform conversion
|
|
670
694
|
success = convert_msi(
|
|
@@ -290,6 +290,17 @@ def convert_msi(
|
|
|
290
290
|
which is the limit SCiLS Lab places on the same
|
|
291
291
|
quantity (2026b User Guide, p.76). Pass None for
|
|
292
292
|
no limit.
|
|
293
|
+
- spectrum_type: str - For imzML, declare the spectrum
|
|
294
|
+
representation explicitly: 'profile' or 'centroid'
|
|
295
|
+
(the full CV names are accepted too). Outranks the
|
|
296
|
+
file's own MS:1000127/MS:1000128, so it can correct a
|
|
297
|
+
file that declares the wrong thing; contradicting a
|
|
298
|
+
declaration is logged as a warning. SCiLS Lab spells
|
|
299
|
+
this --rep_type (2026b User Guide, p.81). Default:
|
|
300
|
+
None, meaning detect. **Changes stored values** for
|
|
301
|
+
files where it disagrees with what detection would
|
|
302
|
+
have chosen, because the representation feeds
|
|
303
|
+
instrument and axis-type selection.
|
|
293
304
|
sparse_format: Sparse matrix format ('csc' or 'csr')
|
|
294
305
|
include_optical: Include optical images (default: True)
|
|
295
306
|
apply_optical_alignment: If True (default) and the MSI source
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Chunk/shard policy for the zarr Thyra writes: rasters and the MSI table.
|
|
2
|
+
|
|
3
|
+
Single source of truth for how Thyra chunks the raster images it writes to
|
|
4
|
+
SpatialData zarr -- and the designated seam for tile-aligned zarr v3 sharding
|
|
5
|
+
once scverse/spatialdata PR #1106 (``raster_write_kwargs``) lands.
|
|
6
|
+
|
|
7
|
+
Cross-repo contract: the viewer (Ousia) morphology tile server streams
|
|
8
|
+
``DEFAULT_TILE_SIZE`` (= 512) edge tiles. Once sharding lands, the zarr INNER
|
|
9
|
+
chunk edge must equal that so one served tile == one decompress. Thyra is a
|
|
10
|
+
separate package and CANNOT import Ousia, so the coupling is DUPLICATED BY
|
|
11
|
+
CONTRACT here -- keep ``INNER_TILE_EDGE`` numerically in sync with Ousia's
|
|
12
|
+
``DEFAULT_TILE_SIZE``; a silent divergence reintroduces cold-tile amplification
|
|
13
|
+
on the viewer. See the Ousia ADR docs/ADR-pyramid-tile-sharding.md.
|
|
14
|
+
|
|
15
|
+
The raster policy above is passed per-array. The TABLE policy below cannot be:
|
|
16
|
+
``SpatialData.write`` hands the table straight to ``anndata.write_elem`` and
|
|
17
|
+
exposes no ``dataset_kwargs`` seam, so the only supported lever is zarr's own
|
|
18
|
+
config. ``table_write_config()`` is that lever, and every writer that reaches
|
|
19
|
+
zarr through spatialdata (the 2D, 3D and streaming-COO converters, all via
|
|
20
|
+
``_save_output``) must hold it open across the write. The streaming-PCS writer
|
|
21
|
+
hand-rolls its ``create_array`` calls and sizes them itself.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from contextlib import contextmanager
|
|
25
|
+
from typing import Iterator, Tuple
|
|
26
|
+
|
|
27
|
+
import zarr
|
|
28
|
+
|
|
29
|
+
# Outer chunk edge used TODAY (pre-sharding the chunk IS the outer block): a 4096
|
|
30
|
+
# block means a viewer 512-tile read decompresses at most one chunk per request.
|
|
31
|
+
IMAGE_CHUNK_EDGE = 4096
|
|
32
|
+
|
|
33
|
+
# Future zarr INNER chunk edge once #1106 lands == Ousia DEFAULT_TILE_SIZE (512).
|
|
34
|
+
# Duplicated by contract (Thyra cannot import Ousia).
|
|
35
|
+
INNER_TILE_EDGE = 512
|
|
36
|
+
|
|
37
|
+
# Uncompressed byte budget for ONE SHARD of a table array -- X/data, X/indices,
|
|
38
|
+
# X/indptr and every obs/var column anndata writes. One shard is one FILE, so
|
|
39
|
+
# this is what keeps a 36M-nnz table (test_data/bellini.imzML) in 11 files
|
|
40
|
+
# instead of 394,699.
|
|
41
|
+
#
|
|
42
|
+
# anndata >= 0.13 turns on ``shards="auto"`` for zarr v3 and would otherwise
|
|
43
|
+
# pick its own 1 GB budget; it explicitly yields to a caller-set value (see
|
|
44
|
+
# ``anndata._io.specs.methods.zarr_v3_sharding``). We set a smaller one because
|
|
45
|
+
# a shard is assembled in memory before it is written -- anndata's own comment
|
|
46
|
+
# says so -- and Thyra converts multi-GB acquisitions on workstations, where a
|
|
47
|
+
# 1 GB write buffer is a real cost. 128 MiB is still enormous relative to the
|
|
48
|
+
# file counts that hurt: test_data/pea.imzML (60.1M nnz, 481 MB of X/data)
|
|
49
|
+
# lands in 4 data files, so even a table 100x that size stays in the hundreds.
|
|
50
|
+
#
|
|
51
|
+
# The INNER chunk stays zarr's ~1 MiB auto pick: that is the unit a random read
|
|
52
|
+
# decompresses, and it is the size the non-sharded path used before anndata
|
|
53
|
+
# began auto-sharding, so read amplification is unchanged.
|
|
54
|
+
TABLE_SHARD_TARGET_BYTES = 128 * 1024 * 1024
|
|
55
|
+
|
|
56
|
+
# zarr < 3.1.4 computed the auto chunk with ``max_bytes=1024`` where 1 MiB was
|
|
57
|
+
# meant, so ``shards="auto"`` produced ~1 KB shards -- one file per KB of table.
|
|
58
|
+
# It also predates ``array.target_shard_size_bytes``, making the budget above
|
|
59
|
+
# unenforceable. pyproject pins the floor at 3.1.4 for exactly this reason;
|
|
60
|
+
# this constant is what the regression test checks the floor still buys us.
|
|
61
|
+
MIN_TABLE_SHARD_BYTES = 64 * 1024
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@contextmanager
|
|
65
|
+
def table_write_config() -> Iterator[None]:
|
|
66
|
+
"""Hold Thyra's table shard budget open across a ``SpatialData.write``.
|
|
67
|
+
|
|
68
|
+
Wraps zarr's global config, so it must stay open for the *whole* write --
|
|
69
|
+
the arrays are created lazily inside anndata, not up front.
|
|
70
|
+
"""
|
|
71
|
+
with zarr.config.set({"array.target_shard_size_bytes": TABLE_SHARD_TARGET_BYTES}):
|
|
72
|
+
yield
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def image_chunks(spatial_ndim: int, edge: int = IMAGE_CHUNK_EDGE) -> Tuple[int, ...]:
|
|
76
|
+
"""The zarr chunk tuple for a Thyra-written raster image (leading channel axis).
|
|
77
|
+
|
|
78
|
+
``spatial_ndim == 2`` (c, y, x) -> ``(1, edge, edge)``;
|
|
79
|
+
``spatial_ndim == 3`` (c, z, y, x) -> ``(1, 1, edge, edge)``.
|
|
80
|
+
|
|
81
|
+
This is the ONE place the future inner=``INNER_TILE_EDGE`` + outer-shard policy
|
|
82
|
+
will live when spatialdata PR #1106 ships ``raster_write_kwargs``. Today it
|
|
83
|
+
returns the pre-sharding behaviour, byte-for-byte.
|
|
84
|
+
"""
|
|
85
|
+
if spatial_ndim == 2:
|
|
86
|
+
return (1, edge, edge)
|
|
87
|
+
if spatial_ndim == 3:
|
|
88
|
+
return (1, 1, edge, edge)
|
|
89
|
+
raise ValueError(f"spatial_ndim must be 2 or 3, got {spatial_ndim}")
|
|
@@ -21,7 +21,7 @@ from ...resampling.gaps import zero_across_gaps
|
|
|
21
21
|
from ...resampling.tic import preserved_tic, rescale_to_preserved_tic
|
|
22
22
|
from ...resampling.types import ResamplingConfig
|
|
23
23
|
from ...utils.zarr_atomic_write import install_windows_atomic_write_retry
|
|
24
|
-
from ._chunking import image_chunks
|
|
24
|
+
from ._chunking import image_chunks, table_write_config
|
|
25
25
|
|
|
26
26
|
logger = logging.getLogger(__name__)
|
|
27
27
|
|
|
@@ -494,56 +494,72 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
|
|
|
494
494
|
except Exception as e:
|
|
495
495
|
logger.debug(f"Could not extract spectrum metadata: {e}")
|
|
496
496
|
|
|
497
|
-
def
|
|
498
|
-
"""
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
-
|
|
503
|
-
|
|
504
|
-
-
|
|
505
|
-
|
|
497
|
+
def build_uns_metadata(self) -> Dict[str, Any]:
|
|
498
|
+
"""The provenance block every write path must persist, identically.
|
|
499
|
+
|
|
500
|
+
Single source of truth for what lands in the table's ``uns``:
|
|
501
|
+
|
|
502
|
+
- ``essential_metadata`` -- dimensions, mass range, source path,
|
|
503
|
+
spectrum type and the Thyra version that wrote the store.
|
|
504
|
+
- ``format_specific`` -- vendor metadata (FlexImaging areas,
|
|
505
|
+
teaching points, imzML file mode, ...).
|
|
506
|
+
- ``acquisition_params`` / ``instrument_info`` -- when the reader
|
|
507
|
+
has them.
|
|
508
|
+
- ``raw_metadata`` -- the source metadata as read.
|
|
509
|
+
- ``regions`` -- the acquisition region summary, as JSON.
|
|
510
|
+
|
|
511
|
+
This exists because the converters have two write paths and they
|
|
512
|
+
drifted. The in-memory and streaming-COO paths hand the table to
|
|
513
|
+
``anndata``'s writer, which serialises whatever is in
|
|
514
|
+
``adata.uns``; the streaming-PCS path hand-writes the Zarr layout
|
|
515
|
+
and used to compose its own, much smaller block -- with
|
|
516
|
+
``spectrum_type`` hardcoded to ``"processed"``, which is not even
|
|
517
|
+
a value the extractors produce. A dataset that crossed
|
|
518
|
+
``StreamingSpatialDataConverter.PCS_SIZE_THRESHOLD_GB`` therefore
|
|
519
|
+
came out claiming a spectrum representation it did not have, and
|
|
520
|
+
without any of the other sections, while a slightly smaller one
|
|
521
|
+
from the same instrument came out complete. Both paths now render
|
|
522
|
+
this mapping, so a section added here reaches every store.
|
|
523
|
+
|
|
524
|
+
Sections the reader has nothing for are omitted rather than
|
|
525
|
+
written empty, so consumers can tell "not available from this
|
|
526
|
+
format" from "available and empty".
|
|
527
|
+
|
|
528
|
+
Returns:
|
|
529
|
+
Mapping of ``uns`` key to the value to store. Empty if the
|
|
530
|
+
reader cannot produce comprehensive metadata at all.
|
|
506
531
|
"""
|
|
507
532
|
try:
|
|
508
533
|
comp_meta = self.reader.get_comprehensive_metadata()
|
|
534
|
+
except Exception as e:
|
|
535
|
+
# Non-fatal: a store without provenance still holds the
|
|
536
|
+
# spectra. But it is not a debug-level event -- the whole
|
|
537
|
+
# point of the block is that a consumer can say where the
|
|
538
|
+
# data came from, so losing it has to be visible in the log.
|
|
539
|
+
logger.warning("Could not read metadata for uns provenance: %s", e)
|
|
540
|
+
return {}
|
|
541
|
+
|
|
542
|
+
uns: Dict[str, Any] = {}
|
|
543
|
+
try:
|
|
544
|
+
self._collect_essential_metadata(uns, comp_meta)
|
|
545
|
+
self._collect_optional_sections(uns, comp_meta)
|
|
546
|
+
self._collect_region_info(uns)
|
|
547
|
+
except Exception as e:
|
|
548
|
+
logger.warning("Could not build the full uns provenance block: %s", e)
|
|
509
549
|
|
|
510
|
-
|
|
511
|
-
self._store_acquisition_params(adata, comp_meta)
|
|
512
|
-
self._store_instrument_info(adata, comp_meta)
|
|
513
|
-
self._store_essential_metadata(adata, comp_meta)
|
|
514
|
-
self._store_raw_metadata(adata, comp_meta)
|
|
515
|
-
self._store_region_info(adata)
|
|
550
|
+
return uns
|
|
516
551
|
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
def _store_format_specific(self, adata, comp_meta) -> None:
|
|
522
|
-
"""Store format-specific metadata (FlexImaging areas, teaching points)."""
|
|
523
|
-
if hasattr(comp_meta, "format_specific") and comp_meta.format_specific:
|
|
524
|
-
adata.uns["format_specific"] = comp_meta.format_specific
|
|
525
|
-
|
|
526
|
-
def _store_acquisition_params(self, adata, comp_meta) -> None:
|
|
527
|
-
"""Store acquisition parameters."""
|
|
528
|
-
if hasattr(comp_meta, "acquisition_params") and comp_meta.acquisition_params:
|
|
529
|
-
adata.uns["acquisition_params"] = comp_meta.acquisition_params
|
|
530
|
-
|
|
531
|
-
def _store_instrument_info(self, adata, comp_meta) -> None:
|
|
532
|
-
"""Store instrument information."""
|
|
533
|
-
if hasattr(comp_meta, "instrument_info") and comp_meta.instrument_info:
|
|
534
|
-
adata.uns["instrument_info"] = comp_meta.instrument_info
|
|
535
|
-
|
|
536
|
-
def _store_essential_metadata(self, adata, comp_meta) -> None:
|
|
537
|
-
"""Store essential metadata (convert tuples to lists for Zarr)."""
|
|
538
|
-
if not hasattr(comp_meta, "essential"):
|
|
552
|
+
def _collect_essential_metadata(self, uns: Dict[str, Any], comp_meta: Any) -> None:
|
|
553
|
+
"""Add ``essential_metadata`` (tuples become lists for Zarr)."""
|
|
554
|
+
essential = getattr(comp_meta, "essential", None)
|
|
555
|
+
if essential is None:
|
|
539
556
|
return
|
|
540
557
|
|
|
541
|
-
essential = comp_meta.essential
|
|
542
558
|
dims = essential.dimensions
|
|
543
559
|
mrange = essential.mass_range
|
|
544
560
|
from thyra import __version__
|
|
545
561
|
|
|
546
|
-
|
|
562
|
+
uns["essential_metadata"] = {
|
|
547
563
|
"source_path": str(essential.source_path),
|
|
548
564
|
"dimensions": list(dims) if dims else None,
|
|
549
565
|
"mass_range": list(mrange) if mrange else None,
|
|
@@ -551,13 +567,19 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
|
|
|
551
567
|
"thyra_version": __version__,
|
|
552
568
|
}
|
|
553
569
|
|
|
554
|
-
def
|
|
555
|
-
"""
|
|
556
|
-
|
|
557
|
-
|
|
570
|
+
def _collect_optional_sections(self, uns: Dict[str, Any], comp_meta: Any) -> None:
|
|
571
|
+
"""Add the vendor sections the reader actually populated."""
|
|
572
|
+
for key in ("format_specific", "acquisition_params", "instrument_info"):
|
|
573
|
+
value = getattr(comp_meta, key, None)
|
|
574
|
+
if value:
|
|
575
|
+
uns[key] = self._serialize_for_zarr(value)
|
|
576
|
+
|
|
577
|
+
raw_metadata = getattr(comp_meta, "raw_metadata", None)
|
|
578
|
+
if raw_metadata:
|
|
579
|
+
uns["raw_metadata"] = self._serialize_for_zarr(raw_metadata)
|
|
558
580
|
|
|
559
|
-
def
|
|
560
|
-
"""
|
|
581
|
+
def _collect_region_info(self, uns: Dict[str, Any]) -> None:
|
|
582
|
+
"""Add the acquisition region summary as JSON.
|
|
561
583
|
|
|
562
584
|
Stored as a JSON string because AnnData/zarr cannot round-trip
|
|
563
585
|
a list of dicts (they get stringified individually). JSON
|
|
@@ -568,7 +590,13 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
|
|
|
568
590
|
"""
|
|
569
591
|
region_info = getattr(self, "_region_info", None)
|
|
570
592
|
if region_info:
|
|
571
|
-
|
|
593
|
+
uns["regions"] = json.dumps(self._serialize_for_zarr(region_info))
|
|
594
|
+
|
|
595
|
+
def _add_metadata_to_uns(self, adata) -> None:
|
|
596
|
+
"""Apply :meth:`build_uns_metadata` to an AnnData about to be written."""
|
|
597
|
+
uns = self.build_uns_metadata()
|
|
598
|
+
adata.uns.update(uns)
|
|
599
|
+
logger.debug("Added MSI metadata to AnnData .uns: %s", sorted(uns))
|
|
572
600
|
|
|
573
601
|
def _serialize_for_zarr(self, obj):
|
|
574
602
|
"""Recursively convert tuples to lists for Zarr serialization."""
|
|
@@ -1921,8 +1949,12 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
|
|
|
1921
1949
|
# Add metadata
|
|
1922
1950
|
self.add_metadata(sdata)
|
|
1923
1951
|
|
|
1924
|
-
# Write to disk
|
|
1925
|
-
|
|
1952
|
+
# Write to disk. table_write_config() must wrap the write itself,
|
|
1953
|
+
# not just the SpatialData construction: anndata creates the table's
|
|
1954
|
+
# zarr arrays lazily inside sdata.write, and that is where the shard
|
|
1955
|
+
# budget is read. Without it the table lands in ~1 KB shards -- one
|
|
1956
|
+
# file per KB -- and the write dominates conversion wall-clock.
|
|
1957
|
+
with _suppress_upstream_warnings(), table_write_config():
|
|
1926
1958
|
sdata.write(str(self.output_path))
|
|
1927
1959
|
zarr.consolidate_metadata(str(self.output_path))
|
|
1928
1960
|
logger.info(f"Successfully saved SpatialData to {self.output_path}")
|
|
@@ -221,6 +221,11 @@ class SpatialData3DConverter(BaseSpatialDataConverter):
|
|
|
221
221
|
per_region[str(r)] = total / max(count, 1)
|
|
222
222
|
adata.uns["average_spectrum_per_region"] = per_region
|
|
223
223
|
|
|
224
|
+
# Add MSI metadata to .uns. The 2D and streaming paths have
|
|
225
|
+
# always done this; the 3D one never did, so a volume came
|
|
226
|
+
# out with no record of where it came from at all.
|
|
227
|
+
self._add_metadata_to_uns(adata)
|
|
228
|
+
|
|
224
229
|
# Make sure region column exists and is correct
|
|
225
230
|
region_key = f"{self.dataset_id}_pixels"
|
|
226
231
|
if "region" not in adata.obs.columns:
|
|
@@ -1296,6 +1296,27 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
|
|
|
1296
1296
|
|
|
1297
1297
|
return mm_indices, mm_data
|
|
1298
1298
|
|
|
1299
|
+
def _write_uns_provenance(self, uns_group: "zarr.Group") -> None:
|
|
1300
|
+
"""Write the shared provenance block into a hand-written ``uns`` group.
|
|
1301
|
+
|
|
1302
|
+
This path composes the Zarr layout itself, so it cannot reach
|
|
1303
|
+
``adata.uns`` the way ``_save_output`` does. It renders the same
|
|
1304
|
+
mapping through anndata's own element writer instead, which is
|
|
1305
|
+
what produces the ``encoding-type`` / ``encoding-version`` pairs
|
|
1306
|
+
``read_lazy`` needs -- getting those right by hand is exactly
|
|
1307
|
+
what this path used to have to do, and exactly where it drifted
|
|
1308
|
+
from the other one.
|
|
1309
|
+
|
|
1310
|
+
Only ``uns`` goes through anndata here. ``obs``/``var`` stay
|
|
1311
|
+
hand-written: they carry the pandas string dtypes this path
|
|
1312
|
+
exists to sidestep, whereas the provenance block is plain dicts,
|
|
1313
|
+
strings, lists and numbers.
|
|
1314
|
+
"""
|
|
1315
|
+
from anndata.io import write_elem
|
|
1316
|
+
|
|
1317
|
+
for key, value in self.build_uns_metadata().items():
|
|
1318
|
+
write_elem(uns_group, key, value)
|
|
1319
|
+
|
|
1299
1320
|
def _write_csc_arrays_to_zarr(
|
|
1300
1321
|
self,
|
|
1301
1322
|
mm_indices: np.memmap,
|
|
@@ -1540,40 +1561,7 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
|
|
|
1540
1561
|
a.attrs["encoding-type"] = "string"
|
|
1541
1562
|
a.attrs["encoding-version"] = "0.2.0"
|
|
1542
1563
|
|
|
1543
|
-
|
|
1544
|
-
em_group.attrs["encoding-type"] = "dict"
|
|
1545
|
-
em_group.attrs["encoding-version"] = "0.1.0"
|
|
1546
|
-
a = em_group.create_array(
|
|
1547
|
-
"dimensions", data=np.asarray(np.array(self._dimensions))
|
|
1548
|
-
)
|
|
1549
|
-
a.attrs["encoding-type"] = "array"
|
|
1550
|
-
a.attrs["encoding-version"] = "0.2.0"
|
|
1551
|
-
a = em_group.create_array(
|
|
1552
|
-
"mass_range",
|
|
1553
|
-
data=np.array([mz_values.min(), mz_values.max()]),
|
|
1554
|
-
)
|
|
1555
|
-
a.attrs["encoding-type"] = "array"
|
|
1556
|
-
a.attrs["encoding-version"] = "0.2.0"
|
|
1557
|
-
a = em_group.create_array(
|
|
1558
|
-
"source_path",
|
|
1559
|
-
data=np.array(str(self.reader.data_path), dtype=str_dtype),
|
|
1560
|
-
)
|
|
1561
|
-
a.attrs["encoding-type"] = "string"
|
|
1562
|
-
a.attrs["encoding-version"] = "0.2.0"
|
|
1563
|
-
a = em_group.create_array(
|
|
1564
|
-
"spectrum_type",
|
|
1565
|
-
data=np.array("processed", dtype=str_dtype),
|
|
1566
|
-
)
|
|
1567
|
-
a.attrs["encoding-type"] = "string"
|
|
1568
|
-
a.attrs["encoding-version"] = "0.2.0"
|
|
1569
|
-
from thyra import __version__
|
|
1570
|
-
|
|
1571
|
-
a = em_group.create_array(
|
|
1572
|
-
"thyra_version",
|
|
1573
|
-
data=np.array(__version__, dtype=str_dtype),
|
|
1574
|
-
)
|
|
1575
|
-
a.attrs["encoding-type"] = "string"
|
|
1576
|
-
a.attrs["encoding-version"] = "0.2.0"
|
|
1564
|
+
self._write_uns_provenance(uns_group)
|
|
1577
1565
|
|
|
1578
1566
|
a = uns_group.create_array("average_spectrum", data=avg_spectrum)
|
|
1579
1567
|
a.attrs["encoding-type"] = "array"
|
|
@@ -9,7 +9,12 @@ from numpy.typing import NDArray
|
|
|
9
9
|
from pyimzml.ImzMLParser import ImzMLParser
|
|
10
10
|
|
|
11
11
|
from ...core.base_extractor import MetadataExtractor
|
|
12
|
-
from ...resampling.constants import
|
|
12
|
+
from ...resampling.constants import (
|
|
13
|
+
BinaryDataType,
|
|
14
|
+
ImzMLAccessions,
|
|
15
|
+
SpectrumType,
|
|
16
|
+
normalize_spectrum_type,
|
|
17
|
+
)
|
|
13
18
|
from ..types import ComprehensiveMetadata, EssentialMetadata
|
|
14
19
|
|
|
15
20
|
logger = logging.getLogger(__name__)
|
|
@@ -18,16 +23,32 @@ logger = logging.getLogger(__name__)
|
|
|
18
23
|
class ImzMLMetadataExtractor(MetadataExtractor):
|
|
19
24
|
"""ImzML-specific metadata extractor with optimized two-phase extraction."""
|
|
20
25
|
|
|
21
|
-
def __init__(
|
|
26
|
+
def __init__(
|
|
27
|
+
self,
|
|
28
|
+
parser: ImzMLParser,
|
|
29
|
+
imzml_path: Path,
|
|
30
|
+
spectrum_type: Optional[str] = None,
|
|
31
|
+
):
|
|
22
32
|
"""Initialize ImzML metadata extractor.
|
|
23
33
|
|
|
24
34
|
Args:
|
|
25
35
|
parser: Initialized ImzML parser
|
|
26
36
|
imzml_path: Path to the ImzML file
|
|
37
|
+
spectrum_type: Optional override for the spectrum representation,
|
|
38
|
+
``"profile"`` or ``"centroid"`` (SCiLS Lab spells this
|
|
39
|
+
``--rep_type``). When given, it wins over anything the file
|
|
40
|
+
says. ``None`` -- the default -- leaves detection alone.
|
|
41
|
+
|
|
42
|
+
Raises:
|
|
43
|
+
ValueError: If ``spectrum_type`` is not a recognised
|
|
44
|
+
representation.
|
|
27
45
|
"""
|
|
28
46
|
super().__init__(parser)
|
|
29
47
|
self.parser = parser
|
|
30
48
|
self.imzml_path = imzml_path
|
|
49
|
+
# Normalised at construction so a typo fails here rather than silently
|
|
50
|
+
# falling through to auto-detection during extraction.
|
|
51
|
+
self.spectrum_type_override = normalize_spectrum_type(spectrum_type)
|
|
31
52
|
|
|
32
53
|
def _extract_essential_impl(self) -> EssentialMetadata:
|
|
33
54
|
"""Extract essential metadata optimized for speed."""
|
|
@@ -524,12 +545,19 @@ class ImzMLMetadataExtractor(MetadataExtractor):
|
|
|
524
545
|
def _detect_centroid_spectrum(self) -> Optional[str]:
|
|
525
546
|
"""Detect spectrum type by looking for MS:1000127 (centroid) or MS:1000128 (profile).
|
|
526
547
|
|
|
527
|
-
|
|
528
|
-
file declares
|
|
548
|
+
An explicit ``spectrum_type`` override wins over everything, so a user
|
|
549
|
+
can correct a file that declares the wrong thing. Otherwise what the
|
|
550
|
+
file *declares* wins, wherever it is written, and only when the file
|
|
551
|
+
declares nothing does Thyra guess -- see
|
|
529
552
|
:meth:`_guess_spectrum_type_from_storage_mode` for what that guess is
|
|
530
553
|
worth.
|
|
531
554
|
"""
|
|
532
555
|
try:
|
|
556
|
+
# 0. An explicit override outranks the file.
|
|
557
|
+
if self.spectrum_type_override is not None:
|
|
558
|
+
self._report_spectrum_type_override()
|
|
559
|
+
return self.spectrum_type_override
|
|
560
|
+
|
|
533
561
|
# 1. The declaration, from metadata the parser already holds.
|
|
534
562
|
result = self._check_parser_metadata_for_spectrum_type()
|
|
535
563
|
if result:
|
|
@@ -547,6 +575,49 @@ class ImzMLMetadataExtractor(MetadataExtractor):
|
|
|
547
575
|
logger.debug(f"Could not detect spectrum type: {e}")
|
|
548
576
|
return None
|
|
549
577
|
|
|
578
|
+
def _report_spectrum_type_override(self) -> None:
|
|
579
|
+
"""Log an override, loudly when it contradicts the file's declaration.
|
|
580
|
+
|
|
581
|
+
Overriding a declared ``MS:1000127``/``MS:1000128`` is a legitimate
|
|
582
|
+
thing to want -- files do declare the wrong representation -- and it is
|
|
583
|
+
also a good way to corrupt a conversion silently. So the two cases are
|
|
584
|
+
not logged alike: agreeing with the file, or overriding a file that
|
|
585
|
+
declares nothing, is INFO; contradicting an explicit declaration is
|
|
586
|
+
WARNING and names both values.
|
|
587
|
+
|
|
588
|
+
This still reads the file's *declarations* (steps 1 and 2) in order to
|
|
589
|
+
compare, but never its guess -- there is nothing informative about
|
|
590
|
+
contradicting a guess. That costs at most the same bounded XML scan
|
|
591
|
+
detection would have done, and only on the opt-in path.
|
|
592
|
+
"""
|
|
593
|
+
override = self.spectrum_type_override
|
|
594
|
+
declared = self._check_parser_metadata_for_spectrum_type()
|
|
595
|
+
if declared is None:
|
|
596
|
+
declared = self._check_xml_for_spectrum_type()
|
|
597
|
+
|
|
598
|
+
if declared is None:
|
|
599
|
+
logger.info(
|
|
600
|
+
"Using spectrum_type override %r; the file declares neither %s "
|
|
601
|
+
"(centroid) nor %s (profile).",
|
|
602
|
+
override,
|
|
603
|
+
ImzMLAccessions.CENTROID_SPECTRUM,
|
|
604
|
+
ImzMLAccessions.PROFILE_SPECTRUM,
|
|
605
|
+
)
|
|
606
|
+
elif declared == override:
|
|
607
|
+
logger.info(
|
|
608
|
+
"Using spectrum_type override %r, which agrees with the file.",
|
|
609
|
+
override,
|
|
610
|
+
)
|
|
611
|
+
else:
|
|
612
|
+
logger.warning(
|
|
613
|
+
"spectrum_type override %r CONTRADICTS the file, which declares "
|
|
614
|
+
"%r. Proceeding with the override -- the stored spectrum type, "
|
|
615
|
+
"and any decision Thyra makes from it, will not match what the "
|
|
616
|
+
"file says about itself. Remove the override to trust the file.",
|
|
617
|
+
override,
|
|
618
|
+
declared,
|
|
619
|
+
)
|
|
620
|
+
|
|
550
621
|
def _check_xml_for_spectrum_type(self) -> Optional[str]:
|
|
551
622
|
"""Check XML for spectrum type markers using streaming parser.
|
|
552
623
|
|
|
@@ -12,6 +12,7 @@ from ...core.base_extractor import MetadataExtractor
|
|
|
12
12
|
from ...core.base_reader import BaseMSIReader
|
|
13
13
|
from ...core.registry import register_reader
|
|
14
14
|
from ...metadata.extractors.imzml_extractor import ImzMLMetadataExtractor
|
|
15
|
+
from ...resampling.constants import normalize_spectrum_type
|
|
15
16
|
|
|
16
17
|
logger = logging.getLogger(__name__)
|
|
17
18
|
|
|
@@ -351,6 +352,14 @@ class ImzMLReader(BaseMSIReader):
|
|
|
351
352
|
``DEFAULT_MAX_MASS_AXIS_LENGTH`` (10 million, SCiLS Lab's own
|
|
352
353
|
limit); pass ``None`` explicitly for unlimited. See
|
|
353
354
|
``_extract_continuous_mass_axis``.
|
|
355
|
+
|
|
356
|
+
``spectrum_type`` declares the spectrum representation
|
|
357
|
+
explicitly -- ``"profile"`` or ``"centroid"`` -- instead of
|
|
358
|
+
letting Thyra read or guess it. SCiLS Lab spells the same thing
|
|
359
|
+
``--rep_type``. Use it when a file declares the wrong
|
|
360
|
+
representation; it outranks the file's own ``MS:1000127`` /
|
|
361
|
+
``MS:1000128``, and contradicting a declaration is logged as a
|
|
362
|
+
warning. Defaults to ``None`` (detect).
|
|
354
363
|
"""
|
|
355
364
|
super().__init__(data_path, **kwargs)
|
|
356
365
|
self.filepath: Optional[Union[str, Path]] = data_path
|
|
@@ -361,6 +370,11 @@ class ImzMLReader(BaseMSIReader):
|
|
|
361
370
|
self.max_mass_axis_length: Optional[int] = kwargs.get(
|
|
362
371
|
"max_mass_axis_length", DEFAULT_MAX_MASS_AXIS_LENGTH
|
|
363
372
|
)
|
|
373
|
+
# Validated here rather than at extraction time, so a bad value fails
|
|
374
|
+
# while the caller is still looking at its own arguments.
|
|
375
|
+
self.spectrum_type: Optional[str] = normalize_spectrum_type(
|
|
376
|
+
kwargs.get("spectrum_type")
|
|
377
|
+
)
|
|
364
378
|
self.parser: Optional[ImzMLParser] = None
|
|
365
379
|
self.ibd_file: Optional[Any] = None
|
|
366
380
|
self.imzml_path: Optional[Path] = None
|
|
@@ -499,7 +513,9 @@ class ImzMLReader(BaseMSIReader):
|
|
|
499
513
|
if not self.imzml_path:
|
|
500
514
|
raise ValueError("ImzML path not available")
|
|
501
515
|
|
|
502
|
-
return ImzMLMetadataExtractor(
|
|
516
|
+
return ImzMLMetadataExtractor(
|
|
517
|
+
self.parser, self.imzml_path, spectrum_type=self.spectrum_type
|
|
518
|
+
)
|
|
503
519
|
|
|
504
520
|
@property
|
|
505
521
|
def has_shared_mass_axis(self) -> bool:
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Constants for resampling and instrument detection."""
|
|
2
|
+
|
|
3
|
+
from typing import Dict, Optional
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class ImzMLAccessions:
|
|
7
|
+
"""PSI-MS and imzML controlled vocabulary accession codes."""
|
|
8
|
+
|
|
9
|
+
# Spectrum type (MS ontology)
|
|
10
|
+
CENTROID_SPECTRUM = "MS:1000127"
|
|
11
|
+
PROFILE_SPECTRUM = "MS:1000128"
|
|
12
|
+
|
|
13
|
+
# Binary data type (imzML ontology)
|
|
14
|
+
CONTINUOUS_BINARY = "IMS:1000030"
|
|
15
|
+
PROCESSED_BINARY = "IMS:1000031"
|
|
16
|
+
|
|
17
|
+
# Software identifiers
|
|
18
|
+
SCILS_LAB = "MS:1002384"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Thresholds:
|
|
22
|
+
"""Threshold values for data classification."""
|
|
23
|
+
|
|
24
|
+
# Peak density threshold for detecting profile data
|
|
25
|
+
# Profile data typically has >5000 points per spectrum
|
|
26
|
+
PROFILE_PEAK_DENSITY = 5000
|
|
27
|
+
|
|
28
|
+
# Dataset size threshold for streaming converter (GB)
|
|
29
|
+
STREAMING_SIZE_GB = 10
|
|
30
|
+
|
|
31
|
+
# Dataset size threshold for PCS method in streaming converter (GB)
|
|
32
|
+
PCS_SIZE_GB = 30
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class SpectrumType:
|
|
36
|
+
"""Spectrum type string constants."""
|
|
37
|
+
|
|
38
|
+
CENTROID = "centroid spectrum"
|
|
39
|
+
PROFILE = "profile spectrum"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# What a caller may write for a spectrum representation, mapped to the CV name
|
|
43
|
+
# Thyra stores. The bare words match SCiLS Lab's ``--rep_type PROFILE|CENTROID``
|
|
44
|
+
# (2026b User Guide p.81); the full CV names are what MS:1000127 / MS:1000128
|
|
45
|
+
# are called, and are what comes back out of the store, so both are accepted.
|
|
46
|
+
SPECTRUM_TYPE_ALIASES: Dict[str, str] = {
|
|
47
|
+
"centroid": SpectrumType.CENTROID,
|
|
48
|
+
"centroided": SpectrumType.CENTROID,
|
|
49
|
+
"centroid spectrum": SpectrumType.CENTROID,
|
|
50
|
+
"profile": SpectrumType.PROFILE,
|
|
51
|
+
"profile spectrum": SpectrumType.PROFILE,
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def normalize_spectrum_type(value: Optional[object]) -> Optional[str]:
|
|
56
|
+
"""Map a user-supplied spectrum representation onto a :class:`SpectrumType`.
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
value: ``"profile"``, ``"centroid"``, either CV name, or ``None``.
|
|
60
|
+
Case and surrounding whitespace are ignored.
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
The matching :class:`SpectrumType` constant, or ``None`` for ``None``.
|
|
64
|
+
|
|
65
|
+
Raises:
|
|
66
|
+
ValueError: If the value is not a recognised representation. A typo
|
|
67
|
+
here would otherwise be indistinguishable from "no override" and
|
|
68
|
+
would silently hand the file back to auto-detection, so this fails
|
|
69
|
+
loudly instead.
|
|
70
|
+
"""
|
|
71
|
+
if value is None:
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
if not isinstance(value, str):
|
|
75
|
+
raise ValueError(
|
|
76
|
+
f"spectrum_type must be a string or None, got {type(value).__name__}"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
key = value.strip().lower()
|
|
80
|
+
if key in SPECTRUM_TYPE_ALIASES:
|
|
81
|
+
return SPECTRUM_TYPE_ALIASES[key]
|
|
82
|
+
|
|
83
|
+
accepted = ", ".join(repr(k) for k in sorted(SPECTRUM_TYPE_ALIASES))
|
|
84
|
+
raise ValueError(f"Unknown spectrum_type {value!r}. Accepted values: {accepted}.")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class BinaryDataType:
|
|
88
|
+
"""Binary data type string constants."""
|
|
89
|
+
|
|
90
|
+
CONTINUOUS = "continuous"
|
|
91
|
+
PROCESSED = "processed"
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
"""Image chunk/shard policy for Thyra-written SpatialData rasters.
|
|
2
|
-
|
|
3
|
-
Single source of truth for how Thyra chunks the raster images it writes to
|
|
4
|
-
SpatialData zarr -- and the designated seam for tile-aligned zarr v3 sharding
|
|
5
|
-
once scverse/spatialdata PR #1106 (``raster_write_kwargs``) lands.
|
|
6
|
-
|
|
7
|
-
Cross-repo contract: the viewer (Ousia) morphology tile server streams
|
|
8
|
-
``DEFAULT_TILE_SIZE`` (= 512) edge tiles. Once sharding lands, the zarr INNER
|
|
9
|
-
chunk edge must equal that so one served tile == one decompress. Thyra is a
|
|
10
|
-
separate package and CANNOT import Ousia, so the coupling is DUPLICATED BY
|
|
11
|
-
CONTRACT here -- keep ``INNER_TILE_EDGE`` numerically in sync with Ousia's
|
|
12
|
-
``DEFAULT_TILE_SIZE``; a silent divergence reintroduces cold-tile amplification
|
|
13
|
-
on the viewer. See the Ousia ADR docs/ADR-pyramid-tile-sharding.md.
|
|
14
|
-
"""
|
|
15
|
-
|
|
16
|
-
from typing import Tuple
|
|
17
|
-
|
|
18
|
-
# Outer chunk edge used TODAY (pre-sharding the chunk IS the outer block): a 4096
|
|
19
|
-
# block means a viewer 512-tile read decompresses at most one chunk per request.
|
|
20
|
-
IMAGE_CHUNK_EDGE = 4096
|
|
21
|
-
|
|
22
|
-
# Future zarr INNER chunk edge once #1106 lands == Ousia DEFAULT_TILE_SIZE (512).
|
|
23
|
-
# Duplicated by contract (Thyra cannot import Ousia).
|
|
24
|
-
INNER_TILE_EDGE = 512
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def image_chunks(spatial_ndim: int, edge: int = IMAGE_CHUNK_EDGE) -> Tuple[int, ...]:
|
|
28
|
-
"""The zarr chunk tuple for a Thyra-written raster image (leading channel axis).
|
|
29
|
-
|
|
30
|
-
``spatial_ndim == 2`` (c, y, x) -> ``(1, edge, edge)``;
|
|
31
|
-
``spatial_ndim == 3`` (c, z, y, x) -> ``(1, 1, edge, edge)``.
|
|
32
|
-
|
|
33
|
-
This is the ONE place the future inner=``INNER_TILE_EDGE`` + outer-shard policy
|
|
34
|
-
will live when spatialdata PR #1106 ships ``raster_write_kwargs``. Today it
|
|
35
|
-
returns the pre-sharding behaviour, byte-for-byte.
|
|
36
|
-
"""
|
|
37
|
-
if spatial_ndim == 2:
|
|
38
|
-
return (1, edge, edge)
|
|
39
|
-
if spatial_ndim == 3:
|
|
40
|
-
return (1, 1, edge, edge)
|
|
41
|
-
raise ValueError(f"spatial_ndim must be 2 or 3, got {spatial_ndim}")
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
"""Constants for resampling and instrument detection."""
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
class ImzMLAccessions:
|
|
5
|
-
"""PSI-MS and imzML controlled vocabulary accession codes."""
|
|
6
|
-
|
|
7
|
-
# Spectrum type (MS ontology)
|
|
8
|
-
CENTROID_SPECTRUM = "MS:1000127"
|
|
9
|
-
PROFILE_SPECTRUM = "MS:1000128"
|
|
10
|
-
|
|
11
|
-
# Binary data type (imzML ontology)
|
|
12
|
-
CONTINUOUS_BINARY = "IMS:1000030"
|
|
13
|
-
PROCESSED_BINARY = "IMS:1000031"
|
|
14
|
-
|
|
15
|
-
# Software identifiers
|
|
16
|
-
SCILS_LAB = "MS:1002384"
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
class Thresholds:
|
|
20
|
-
"""Threshold values for data classification."""
|
|
21
|
-
|
|
22
|
-
# Peak density threshold for detecting profile data
|
|
23
|
-
# Profile data typically has >5000 points per spectrum
|
|
24
|
-
PROFILE_PEAK_DENSITY = 5000
|
|
25
|
-
|
|
26
|
-
# Dataset size threshold for streaming converter (GB)
|
|
27
|
-
STREAMING_SIZE_GB = 10
|
|
28
|
-
|
|
29
|
-
# Dataset size threshold for PCS method in streaming converter (GB)
|
|
30
|
-
PCS_SIZE_GB = 30
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
class SpectrumType:
|
|
34
|
-
"""Spectrum type string constants."""
|
|
35
|
-
|
|
36
|
-
CENTROID = "centroid spectrum"
|
|
37
|
-
PROFILE = "profile spectrum"
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
class BinaryDataType:
|
|
41
|
-
"""Binary data type string constants."""
|
|
42
|
-
|
|
43
|
-
CONTINUOUS = "continuous"
|
|
44
|
-
PROCESSED = "processed"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|