thyra 4.2.0__tar.gz → 4.3.0__tar.gz

{thyra-4.2.0 → thyra-4.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: thyra
-Version: 4.2.0
+Version: 4.3.0
 Summary: A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics
 Project-URL: Homepage, https://github.com/M4i-Imaging-Mass-Spectrometry/thyra
 Project-URL: Repository, https://github.com/M4i-Imaging-Mass-Spectrometry/thyra

{thyra-4.2.0 → thyra-4.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "thyra"
-version = "4.2.0"
+version = "4.3.0"
 description = "A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics"
 authors = [{ name = "Theodoros Visvikis", email = "t.visvikis@maastrichtuniversity.nl" }]
 maintainers = [{ name = "Theodoros Visvikis", email = "t.visvikis@maastrichtuniversity.nl" }]

{thyra-4.2.0 → thyra-4.3.0}/thyra/__init__.py

@@ -67,7 +67,7 @@ warnings.filterwarnings(
     category=FutureWarning,
 )
 
-__version__ = "4.2.0"
+__version__ = "4.3.0"
 
 #: Public name -> the module that defines it, imported on first access.
 _LAZY_ATTRS = {

{thyra-4.2.0 → thyra-4.3.0}/thyra/__main__.py

@@ -331,7 +331,14 @@ def _display_calibration_info(input: Path, use_recalibrated: bool) -> None:
             f"{'s' if n_recalibrations > 1 else ''} since acquisition"
         )
     for state in states:
-        active_marker = " (active/will be used)" if state["id"] == active_id else ""
+        # With --no-recalibrated, Bruker's library applies none of the
+        # states, the newest included: it takes the analysis database's
+        # own calibration instead.
+        active_marker = (
+            " (active/will be used)"
+            if use_recalibrated and state["id"] == active_id
+            else ""
+        )
         click.echo(
             f"  State {state['id']}: {state['datetime']} "
             f"[{state['source']}]{active_marker}"
@@ -340,7 +347,10 @@ def _display_calibration_info(input: Path, use_recalibrated: bool) -> None:
     if use_recalibrated:
         click.echo(f"\nUsing active calibration state (State {active_id})")
     else:
-        click.echo("\nUsing original calibration (--no-recalibrated flag set)")
+        click.echo(
+            "\nUsing original calibration (--no-recalibrated flag set): "
+            "none of these states"
+        )
 
     click.echo(
         "\nNote: this lists the states only; one cannot be chosen here. "

{thyra-4.2.0 → thyra-4.3.0}/thyra/converters/spatialdata/streaming_converter.py

@@ -51,21 +51,27 @@ from tqdm import tqdm
 
 from ...core.conversion_state import ConversionState
 from ...errors import ConversionRefused
+from ...metadata.schema import MSI_OBS_ACQUISITION_ORDER_COLUMN
 from .base_spatialdata_converter import BaseSpatialDataConverter
 from .csc_assembly import CscAssembly, index_dtype
 
 logger = logging.getLogger(__name__)
 
+#: The acquisition order of a grid position no spectrum has reached yet:
+#: larger than any real one, so the first spectrum's order replaces it.
+_NOT_ACQUIRED = np.iinfo(np.int64).max
+
 
 class _TableUnit:
     """One table of the store in the making: a z plane, or the whole volume.
 
     Holds what the two passes accumulate for it -- the column counts and
     the scattered matrix (``assembly``), which grid positions carry a
-    spectrum (``occupancy``) and the TIC raster -- and, once pass 1 is
-    done, the row layout: ``kept_grid`` is the grid index of each table
-    row and ``row_of_grid`` the table row of each grid position (-1 when
-    dropped as empty).
+    spectrum (``occupancy``), the TIC raster and, when the reader knows
+    it, each position's acquisition order -- and, once pass 1 is done,
+    the row layout: ``kept_grid`` is the grid index of each table row and
+    ``row_of_grid`` the table row of each grid position (-1 when dropped
+    as empty).
 
     Args:
         key: The table's element key (``{id}_z{z}`` or ``{id}``).
@@ -75,6 +81,8 @@ class _TableUnit:
         n_cols: Length of the mass axis.
         tic_shape: ``(n_y, n_x)`` for a plane, ``(n_z, n_y, n_x)`` for a
             volume.
+        records_order: Whether every spectrum comes with its acquisition
+            order (the reader's ``has_acquisition_order``).
     """
 
     def __init__(
@@ -85,6 +93,7 @@ class _TableUnit:
         n_grid: int,
         n_cols: int,
         tic_shape: Tuple[int, ...],
+        records_order: bool = False,
     ) -> None:
         """Allocate the pass-1 accumulators; the matrix comes after pass 1."""
         self.key = key
@@ -104,14 +113,31 @@ class _TableUnit:
         #: coordinate to name if it does not come out as pass 1's.
         self.observed: Dict[int, Tuple[float, Tuple[int, int, int]]] = {}
         self.tic = np.zeros(tic_shape, dtype=np.float64)
+        #: Per grid position, the smallest acquisition order among the
+        #: spectra it got; ``None`` when the reader does not know the
+        #: order. A position no spectrum reached keeps the sentinel and
+        #: never becomes a row, so no row can read it.
+        self.acquisition_order: Optional[NDArray[np.int64]] = (
+            np.full(self.n_grid, _NOT_ACQUIRED, dtype=np.int64)
+            if records_order
+            else None
+        )
         self.kept_grid: Optional[NDArray[np.int64]] = None
         self.row_of_grid: Optional[NDArray[np.int64]] = None
         self.n_rows = 0
 
     def count(
-        self, grid: int, mz_indices: NDArray[np.int_], values: NDArray[np.float64]
+        self,
+        grid: int,
+        mz_indices: NDArray[np.int_],
+        values: NDArray[np.float64],
+        order: Optional[int] = None,
     ) -> None:
-        """Pass 1: one spectrum at grid position ``grid``."""
+        """Pass 1: one spectrum at grid position ``grid``.
+
+        ``order`` is the spectrum's acquisition order, required when the
+        unit records one.
+        """
         self.assembly.count(grid, mz_indices)
         if self.occupancy[grid]:
             # A pixel the source measured twice. One position is one row,
@@ -122,6 +148,16 @@ class _TableUnit:
             self.assembly.merge_duplicates = True
         self.occupancy[grid] = True
         self.tic.reshape(-1)[grid] += values.sum()
+        if self.acquisition_order is not None:
+            if order is None:
+                raise ValueError(
+                    f"{self.key}: a spectrum reached the table without its "
+                    "acquisition order"
+                )
+            # A row summed from two measurements was first measured at the
+            # earlier one, whatever order the reader yields them in.
+            if order < self.acquisition_order[grid]:
+                self.acquisition_order[grid] = order
 
     def finish_counting(self) -> None:
         """Decide which grid positions become rows, and in what order.
@@ -436,6 +472,7 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
             raise ValueError("Common mass axis is not initialized")
         n_x, n_y, n_z = self._dimensions
         n_cols = len(self._common_mass_axis)
+        records_order = bool(self.reader.has_acquisition_order)
         if self.handle_3d:
             return [
                 _TableUnit(
@@ -445,6 +482,7 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
                     n_x * n_y * n_z,
                     n_cols,
                     (n_z, n_y, n_x),
+                    records_order,
                 )
             ]
         return [
@@ -455,6 +493,7 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
                 n_x * n_y,
                 n_cols,
                 (n_y, n_x),
+                records_order,
             )
             for z in range(n_z)
         ]
@@ -533,22 +572,36 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
             return units[0], z * n_x * n_y + y * n_x + x
         return units[z], y * n_x + x
 
-    def _iter_pass_spectra(
-        self, passes: Any, phase: str
-    ) -> Generator[Tuple[Tuple[int, int, int], Any, Any, Any], None, None]:
+    def _iter_pass_spectra(self, passes: Any, phase: str) -> Generator[
+        Tuple[Tuple[int, int, int], Optional[int], Any, Any, Any],
+        None,
+        None,
+    ]:
         """The summed spectra of one pass, with the frame record they came from.
 
-        Yields ``(coords, mzs, intensities, frame)``: ``frame`` is the
-        record the sinks are fed from, or ``None`` when the pass reads
-        the summed spectra directly (no sinks, or a reader without
-        records). A frame whose summed spectrum is empty is not yielded,
-        exactly as ``iter_spectra`` never yields it, but the sinks of
-        ``phase`` (``"count"`` or ``"scatter"``) still see it with no row,
-        which is what their own pass would have shown them.
+        Yields ``(coords, order, mzs, intensities, frame)``: ``order`` is
+        the spectrum's acquisition order, or ``None`` when the reader does
+        not know it, and ``frame`` is the record the sinks are fed from,
+        or ``None`` when the pass reads the summed spectra directly (no
+        sinks, or a reader without records). A frame whose summed spectrum
+        is empty is not yielded, exactly as ``iter_spectra`` never yields
+        it, but the sinks of ``phase`` (``"count"`` or ``"scatter"``) still
+        see it with no row, which is what their own pass would have shown
+        them.
         """
+        ordered = bool(self.reader.has_acquisition_order)
         if passes is None or passes.empty:
+            if ordered:
+                for (
+                    coords,
+                    order,
+                    mzs,
+                    intensities,
+                ) in self.reader.iter_spectra_with_acquisition_order():
+                    yield coords, order, mzs, intensities, None
+                return
             for coords, mzs, intensities in self.reader.iter_spectra():
-                yield coords, mzs, intensities, None
+                yield coords, None, mzs, intensities, None
             return
         feed = getattr(passes, phase)
         for frame in self.reader.iter_frame_scans():
@@ -556,7 +609,8 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
             if spectrum is None:
                 feed(frame, None)
                 continue
-            yield frame.coords, spectrum[0], spectrum[1], frame
+            order = frame.acquisition_order if ordered else None
+            yield frame.coords, order, spectrum[0], spectrum[1], frame
 
     def _process_spectra(self, state: ConversionState) -> None:
         """Run both passes: count, size the arrays, scatter.
@@ -762,7 +816,7 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
             desc="Pre-scan" if passes is None else "Pre-scan + sibling tables",
             unit="spectrum",
         ) as pbar:
-            for coords, mzs, intensities, frame in self._iter_pass_spectra(
+            for coords, order, mzs, intensities, frame in self._iter_pass_spectra(
                 passes, "count"
             ):
                 x, y, z = coords
@@ -791,7 +845,7 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
                     # never writes -- explicit zeros, out of order, which
                     # scipy reports as a non-canonical matrix.
                     if unit is not None:
-                        unit.count(grid, mz_indices, values)
+                        unit.count(grid, mz_indices, values, order)
 
                         # Behind the same check, and for the same reason
                         # the per-table average is taken over rows: the
@@ -876,7 +930,7 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
             desc="Scatter to CSC" if passes is None else "Scatter to CSC + siblings",
             unit="spectrum",
         ) as pbar:
-            for coords, mzs, intensities, frame in self._iter_pass_spectra(
+            for coords, _, mzs, intensities, frame in self._iter_pass_spectra(
                 passes, "scatter"
             ):
                 x, y, z = coords
@@ -1010,6 +1064,9 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
         the in-plane pitch so the table lands in the same micrometre frame
         as the TIC volume's ``Scale``. Column order and dtypes are the ones
         the two in-memory converters wrote, so a store reads back the same.
+        ``acquisition_order`` follows, int64, when the reader knows the
+        order (design decision D26); the rows stay in grid order either
+        way.
         """
         if unit.kept_grid is None or self._dimensions is None:
             raise RuntimeError("The row layout is not decided yet")
@@ -1053,6 +1110,11 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
         obs.set_index("instance_id", inplace=True)
         # Always add per-pixel region numbers for a consistent schema.
         obs["region_number"] = self.build_region_numbers(x_idx, y_idx)
+        # Indexed by the same kept grid as every column above, so a row's
+        # order is its own position's, region filtering included: the
+        # filter removes frames before pass 1 ever sees them.
+        if unit.acquisition_order is not None:
+            obs[MSI_OBS_ACQUISITION_ORDER_COLUMN] = unit.acquisition_order[kept]
         return obs
 
     def _tic_image(self, unit: _TableUnit) -> Any:

{thyra-4.2.0 → thyra-4.3.0}/thyra/core/base_reader.py

@@ -319,6 +319,54 @@ class BaseMSIReader(ABC):
         """
         return None
 
+    # ------------------------------------------------------------------
+    # Acquisition order
+    #
+    # When a spectrum was recorded, relative to the others. ``iter_spectra``
+    # says where each spectrum sits and nothing about when, and the rows a
+    # converter writes follow the grid, which a serpentine scan or a slide
+    # of several regions does not. A drift plotted against the rows is
+    # plotted against raster order.
+    # ------------------------------------------------------------------
+
+    @property
+    def has_acquisition_order(self) -> bool:
+        """Whether the source says in which order its spectra were acquired."""
+        return False
+
+    def iter_spectra_with_acquisition_order(self) -> Generator[
+        Tuple[
+            Tuple[int, int, int],
+            int,
+            NDArray[np.float64],
+            NDArray[np.float64],
+        ],
+        None,
+        None,
+    ]:
+        """:meth:`iter_spectra`, with each spectrum's acquisition order.
+
+        Yields ``((x, y, z), order, mzs, intensities)``: the spectra
+        :meth:`iter_spectra` yields, in its order, with its coordinates and
+        arrays. ``order`` grows with acquisition time and no two spectra
+        share one. It is the source's own number for the spectrum where the
+        source numbers them (a Bruker ``Frames.Id``), else the spectrum's
+        0-based position in the sequence the source lists them in. So it
+        need not start at 0 or be contiguous: sort by it, never index with
+        it.
+
+        The converter reads the spectra through this method whenever
+        :attr:`has_acquisition_order` is True, so a subclass that changes
+        what :meth:`iter_spectra` yields has to change this as well.
+
+        Raises:
+            NotImplementedError: When :attr:`has_acquisition_order` is False.
+        """
+        raise NotImplementedError(
+            f"{type(self).__name__} does not know the order its spectra were "
+            "acquired in"
+        )
+
     # ------------------------------------------------------------------
     # One read per frame for every table
     #

{thyra-4.2.0 → thyra-4.3.0}/thyra/core/frames.py

@@ -46,6 +46,13 @@ class FrameScans(Protocol):
 
     coords: Coords
 
+    @property
+    def acquisition_order(self) -> Optional[int]:
+        """The frame's order as ``iter_spectra_with_acquisition_order`` yields it.
+
+        ``None`` when the reader does not know the order.
+        """
+
     def spectrum(self) -> Optional[Spectrum]:
         """``(mzs, intensities)`` as :meth:`BaseMSIReader.iter_spectra` yields, or ``None``.
 

{thyra-4.2.0 → thyra-4.3.0}/thyra/metadata/extractors/imzml_extractor.py

@@ -33,6 +33,18 @@ UM_PER_UNIT = {
     "UO:0000018": 0.001,  # nanometer
 }
 
+# The name IMS:1000046 carried until imagingMS.obo commit 421481e of
+# 2017-09-07. Under that name the value is the AREA of a pixel, and
+# IMS:1000047 is "image shape", which is not a size at all. Files written
+# under the old vocabulary are still published.
+OLD_PIXEL_SIZE_NAME = "pixel size"
+
+
+def has_old_pixel_size_name(name: object) -> bool:
+    """Whether IMS:1000046 is spelled as it was when it gave an area."""
+    return isinstance(name, str) and name.strip().lower() == OLD_PIXEL_SIZE_NAME
+
+
 # Analyzer-component terms, in preference order. The first one found under a
 # <componentList><analyzer> is surfaced as ``instrument_info["analyzer"]``, so
 # on a hybrid that declares several analyzers the highest-resolution stage --
@@ -533,12 +545,21 @@ class ImzMLMetadataExtractor(MetadataExtractor):
         nanometre pixel size as micrometres, 1000x too large, and
         ``convert_msi`` still returned ``True``. The unit survives on the
         ParamGroup path, so it is read from there and the value converted.
+
+        The name the file gave the term survives there too. Under its old
+        name the value is an area, so it is not read, see
+        :meth:`_is_old_pixel_size`.
         """
         if hasattr(self.parser, "imzmldict") and self.parser.imzmldict:
             # Check for pixel size parameters in the parsed dictionary
             x_size = self.parser.imzmldict.get("pixel size x")
             y_size = self.parser.imzmldict.get("pixel size y")
 
+            if x_size is not None and self._is_old_pixel_size(
+                self._pixel_size_raw_name(), x_size
+            ):
+                return None
+
             if x_size is not None and y_size is not None:
                 try:
                     x_size, y_size = float(x_size), float(y_size)
@@ -582,6 +603,44 @@ class ImzMLMetadataExtractor(MetadataExtractor):
                     units[accession] = unit_accession
         return units
 
+    def _pixel_size_raw_name(self) -> Optional[str]:
+        """The name the file itself gives IMS:1000046, or ``None``.
+
+        pyimzml renames the term to its current name while parsing, so
+        ``name`` always reads "pixel size (x)". ``raw_name`` is what the
+        document says. First declaration wins, as for the unit.
+        """
+        metadata = getattr(self.parser, "metadata", None)
+        scan_settings = getattr(metadata, "scan_settings", None)
+        if not isinstance(scan_settings, dict):
+            return None
+
+        for group in scan_settings.values():
+            for param in getattr(group, "cv_params", []):
+                if param[1] == ImzMLAccessions.PIXEL_SIZE_X:
+                    raw_name = param[3]
+                    return raw_name if isinstance(raw_name, str) else None
+        return None
+
+    def _is_old_pixel_size(self, name: object, value: object) -> bool:
+        """Whether IMS:1000046 is written under its name of before 2017.
+
+        Under that name the value is the area of a pixel. Read as a length
+        it is wrong by its own square root: 10000 for a 100 um pixel. So it
+        is not read, the log says why, and the conversion asks for
+        ``--pixel-size``.
+        """
+        if not has_old_pixel_size_name(name):
+            return False
+        logger.warning(
+            '%s names IMS:1000046 "pixel size", as the vocabulary did before '
+            "2017, when the value (%r) was the area of a pixel. It is not "
+            "read as a pixel size. Pass --pixel-size.",
+            Path(self.imzml_path).name,
+            value,
+        )
+        return True
+
     def _pixel_size_to_um(self, value: float, unit_accession: Optional[str]) -> float:
         """Convert a declared pixel size to micrometres, or refuse.
 
@@ -1170,11 +1229,15 @@ class ImzMLMetadataExtractor(MetadataExtractor):
         The unit conversion happens outside the ``try`` so a refused unit
         propagates as loudly as it does on the fast path, while a merely
         unparseable document still degrades to "no pixel size".
+
+        IMS:1000046 under its old name is not read here either, see
+        :meth:`_is_old_pixel_size`.
         """
         x_size = None
         y_size = None
         x_unit = None
         y_unit = None
+        x_name = None
 
         try:
             if not hasattr(self.parser, "metadata") or not hasattr(
@@ -1196,6 +1259,7 @@ class ImzMLMetadataExtractor(MetadataExtractor):
                 if accession == ImzMLAccessions.PIXEL_SIZE_X:
                     x_size = float(cvparam.get("value", 0))
                     x_unit = cvparam.get("unitAccession")
+                    x_name = cvparam.get("name")
                 elif accession == ImzMLAccessions.PIXEL_SIZE_Y:
                     y_size = float(cvparam.get("value", 0))
                     y_unit = cvparam.get("unitAccession")
@@ -1204,6 +1268,9 @@ class ImzMLMetadataExtractor(MetadataExtractor):
             logger.warning(f"Failed to parse XML metadata for pixel size: {e}")
             return None
 
+        if x_size is not None and self._is_old_pixel_size(x_name, x_size):
+            return None
+
         if x_size is not None and y_size is not None:
             x_um = self._pixel_size_to_um(x_size, x_unit)
             y_um = self._pixel_size_to_um(y_size, y_unit)