thyra 3.2.1__tar.gz → 3.3.0__tar.gz

{thyra-3.2.1 → thyra-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thyra
-Version: 3.2.1
+Version: 3.3.0
 Summary: A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics
 License: MIT
 License-File: LICENSE
@@ -81,7 +81,7 @@ for a full walkthrough, including the published example dataset
 
 ## Features
 
-- **Multiple Input Formats**: ImzML, Bruker (.d directories), Waters (.raw directories)
+- **Multiple Input Formats**: ImzML, Bruker (.d directories), Waters (.raw directories), PHI SmartSoft-TOF ToF-SIMS (.raw files)
 - **SpatialData Output**: Modern, cloud-ready format with Zarr backend
 - **Memory Efficient**: Handles large datasets (100+ GB) through streaming processing
 - **Optical Alignment**: Automatic MSI-to-optical image registration for Bruker data
@@ -107,10 +107,17 @@ thyra input.imzML output.zarr
 # Bruker data with verbose logging
 thyra data.d output.zarr -v DEBUG
 
+# PHI SmartSoft-TOF ToF-SIMS (a .raw file, not a directory)
+thyra tofsims_run.raw output.zarr
+
 # Disable resampling
 thyra input.imzML output.zarr --no-resample
 ```
 
+Thyra auto-detects the input format. Note that `.raw` is claimed by two
+vendors and resolved by shape: Waters writes a directory, PHI writes a single
+file. See [Supported Formats](https://M4i-Imaging-Mass-Spectrometry.github.io/thyra/supported-formats/).
+
 ### Python API
 
 ```python

{thyra-3.2.1 → thyra-3.3.0}/README.md

@@ -27,7 +27,7 @@ for a full walkthrough, including the published example dataset
 
 ## Features
 
-- **Multiple Input Formats**: ImzML, Bruker (.d directories), Waters (.raw directories)
+- **Multiple Input Formats**: ImzML, Bruker (.d directories), Waters (.raw directories), PHI SmartSoft-TOF ToF-SIMS (.raw files)
 - **SpatialData Output**: Modern, cloud-ready format with Zarr backend
 - **Memory Efficient**: Handles large datasets (100+ GB) through streaming processing
 - **Optical Alignment**: Automatic MSI-to-optical image registration for Bruker data
@@ -53,10 +53,17 @@ thyra input.imzML output.zarr
 # Bruker data with verbose logging
 thyra data.d output.zarr -v DEBUG
 
+# PHI SmartSoft-TOF ToF-SIMS (a .raw file, not a directory)
+thyra tofsims_run.raw output.zarr
+
 # Disable resampling
 thyra input.imzML output.zarr --no-resample
 ```
 
+Thyra auto-detects the input format. Note that `.raw` is claimed by two
+vendors and resolved by shape: Waters writes a directory, PHI writes a single
+file. See [Supported Formats](https://M4i-Imaging-Mass-Spectrometry.github.io/thyra/supported-formats/).
+
 ### Python API
 
 ```python

{thyra-3.2.1 → thyra-3.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "thyra"
-version = "3.2.1"
+version = "3.3.0"
 description = "A modern Python library for converting Mass Spectrometry Imaging (MSI) data into SpatialData/Zarr format - your portal to spatial omics"
 authors = ["Theodoros Visvikis <t.visvikis@maastrichtuniversity.nl>"]
 maintainers = ["Theodoros Visvikis <t.visvikis@maastrichtuniversity.nl>"]

{thyra-3.2.1 → thyra-3.3.0}/thyra/__init__.py

@@ -36,7 +36,7 @@ warnings.filterwarnings(
     category=FutureWarning,
 )
 
-__version__ = "3.2.1"
+__version__ = "3.3.0"
 
 # Import key components - avoid wildcard imports
 try:

{thyra-3.2.1 → thyra-3.3.0}/thyra/converters/spatialdata/base_spatialdata_converter.py

@@ -151,7 +151,7 @@ try:
     import xarray as xr
     import zarr
     from anndata import AnnData
-    from shapely.geometry import Polygon, box
+    from shapely.geometry import box
     from spatialdata import SpatialData
     from spatialdata.models import Image2DModel, ShapesModel, TableModel
     from spatialdata.transformations import Affine, Identity, Scale, Sequence
@@ -614,9 +614,14 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
         logger.debug("Added MSI metadata to AnnData .uns: %s", sorted(uns))
 
     def _serialize_for_zarr(self, obj):
-        """Recursively convert tuples to lists for Zarr serialization."""
+        """Recursively convert tuples to lists for Zarr serialization.
+
+        Dict keys are coerced to strings: Zarr group members must be named,
+        and a non-string key otherwise fails at write time, after the whole
+        conversion has already been done.
+        """
         if isinstance(obj, dict):
-            return {k: self._serialize_for_zarr(v) for k, v in obj.items()}
+            return {str(k): self._serialize_for_zarr(v) for k, v in obj.items()}
         elif isinstance(obj, (list, tuple)):
             return [self._serialize_for_zarr(item) for item in obj]
         elif hasattr(obj, "__dict__"):
@@ -1434,11 +1439,64 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
         if self._common_mass_axis is None:
             raise ValueError("Common mass axis is not initialized")
 
+        n_channels = len(self._common_mass_axis)
+        columns: Dict[str, Any] = {"mz": self._common_mass_axis}
+        columns.update(self._validated_mass_axis_annotations())
+
         return pd.DataFrame(
-            {"mz": self._common_mass_axis},
-            index=[f"mz_{i}" for i in range(len(self._common_mass_axis))],
+            columns,
+            index=[f"mz_{i}" for i in range(n_channels)],
         )
 
+    def _validated_mass_axis_annotations(self) -> Dict[str, Any]:
+        """Reader-supplied per-channel columns that fit the axis being written.
+
+        A reader whose native axis is not m/z (flight time, drift time) can
+        keep that axis alongside ``mz`` so the conversion stays reversible.
+        Annotations whose length does not match are dropped rather than
+        raising: that is the expected outcome when resampling rebuilds the
+        axis, and it must not fail an otherwise good conversion.
+        """
+        if self._common_mass_axis is None:
+            return {}
+        n_channels = len(self._common_mass_axis)
+
+        # getattr rather than a bare call: readers predating this hook, and
+        # test doubles that do not subclass BaseMSIReader, simply have nothing
+        # to contribute and should not have to raise to say so.
+        getter = getattr(self.reader, "get_mass_axis_annotations", None)
+        if getter is None:
+            return {}
+
+        try:
+            annotations = getter()
+        except Exception as exc:  # pragma: no cover - reader-defined
+            # Format eagerly. Passing the exception itself to the logger keeps
+            # it alive inside the log record, and with it its traceback, the
+            # caller frames reachable through tb_frame.f_back, and any memmap
+            # those frames hold -- which on Windows leaves the backing file
+            # locked long after the conversion has finished.
+            detail = f"{type(exc).__name__}: {exc}"
+            logger.warning("Reader failed to supply mass axis annotations: %s", detail)
+            return {}
+
+        validated: Dict[str, Any] = {}
+        for name, values in (annotations or {}).items():
+            if name == "mz":
+                logger.warning("Ignoring mass axis annotation named 'mz'")
+                continue
+            if len(values) != n_channels:
+                logger.info(
+                    "Dropping mass axis annotation %r: %d values for a "
+                    "%d channel axis (expected when resampling rebuilds it)",
+                    name,
+                    len(values),
+                    n_channels,
+                )
+                continue
+            validated[name] = values
+        return validated
+
     def _get_pixel_index(self, x: int, y: int, z: int) -> int:
         """Calculate linear pixel index from 3D coordinates.
 
@@ -1509,48 +1567,63 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
 
         coo_arrays["current_idx"] = end_idx
 
-    def _create_pixel_shapes(
-        self, adata: AnnData, is_3d: bool = False
-    ) -> "ShapesModel":
+    def _create_pixel_shapes(self, adata: AnnData) -> "ShapesModel":
         """Create geometric shapes for pixels with proper transformations.
 
         When optical alignment is available (FlexImaging with Area definitions),
         shapes are created in optical image pixel coordinates for proper overlay.
         Otherwise, shapes use physical (micrometer) coordinates.
 
-        For a multi-slice volume the footprints are ``POLYGON Z``: a flat
-        square at the micrometre depth of the slice it was acquired on,
-        taken from ``obs["spatial_z"]``. They are footprints, not voxels --
-        shapely has no solid, so a pixel is its square at one depth rather
-        than a box spanning the slice thickness.
-
-        Two alternatives were measured and rejected:
-
+        **Footprints are two-dimensional, including on a multi-slice
+        volume.** A slice's depth is carried by the TIC image's ``Scale``
+        and by ``obs["spatial_z"]``; it is deliberately not also put on
+        the polygon geometry.
+
+        Three options were measured. All three are imperfect, so what
+        follows is the reasoning rather than a claim that this one is
+        free:
+
+        * **``POLYGON Z``** -- geometrically the most honest, and what
+          7317792 shipped in v3.2.0. It breaks ``spatialdata``'s spatial
+          queries: measured on a 5x3x2 volume, a bounding box enclosing
+          the whole dataset with 1000um of margin returned 26 of 30
+          footprints and 26 of 30 table rows, with no exception and no
+          warning. ``shapely.force_2d`` on the same geometry restored
+          30 of 30. A z-restricted query returned the same rows whether
+          z was inside or far outside the data, so z is ignored by the
+          query path rather than honoured.
         * **Flat 2D shapes carrying the depth in a transformation**
-          (one element per slice, each with a ``Translation`` in z). This
-          is what spatialdata's 2D shapes model invites, and it does not
-          work: the element parses without complaint, coexists with a 3D
-          image and round-trips, but the transform **silently drops z**
-          and every slice comes back at the same depth. It would put a
-          depth in the metadata that never reaches the geometry.
-        * **Leaving them flat** and documenting it. Honest, but then
-          ``docs/coordinate-systems.md``'s promise that every element
-          agrees at ``"global"`` stays false in z.
-
-        The cost of the choice: ``ShapesModel.parse``/``validate`` and
-        every downstream ``transform`` emit a ``UserWarning`` that 2
-        dimensions are expected. It is left unsuppressed deliberately --
-        it is upstream telling consumers this is outside the model, and
-        hiding it would be Thyra deciding that on their behalf. Only the
-        micrometre branch gains z; under optical alignment the frame is
-        optical pixels, which have no depth calibration to place a slice
-        in.
+          (one element per slice, each with a ``Translation`` in z).
+          The element parses, coexists with a 3D image and round-trips,
+          but the transform **silently drops z** and every slice comes
+          back at the same depth -- a depth in the metadata that never
+          reaches the geometry. ``test_3d_pixel_shapes_z.py`` pins this,
+          because it is the change someone will otherwise propose.
+        * **Flat, with the depth on the image and in ``obs`` only** --
+          what this does.
+
+        The deciding argument is that ``spatialdata`` itself asks for
+        this. ``ShapesModel.validate`` warns that a 3-dimensional
+        geometry column "could led to unexpected behaviors" and names
+        ``force_2d()`` as the remedy; the query result above is that
+        behaviour. 3D shapes are not on the upstream roadmap
+        (scverse/spatialdata#109 has been idle since June 2023 and
+        covers images, labels and transformations only), and the live
+        2.5D discussion (#961) scopes itself to points, images and
+        labels. Serial-section MSI is 2.5D in that taxonomy.
+
+        What this costs: ``docs/coordinate-systems.md``'s promise that
+        every element agrees at ``"global"`` is exact in x and y and
+        silent in z for the shapes element. That is a documented gap
+        rather than a wrong answer, which a truncated query is not. See
+        ``docs/output-format.md`` for the consumer-facing statement.
+
+        Reinstating z means restoring the ``is_3d`` parameter at all
+        three call sites as well; it was removed with the geometry so it
+        could not sit unread, which is how the original defect arose.
 
         Args:
             adata: AnnData object containing coordinates
-            is_3d: Whether this is the 3D volume route. Combined with
-                :attr:`_is_volume` (which also requires more than one
-                plane), decides whether footprints carry z.
 
         Returns:
             SpatialData shapes model
@@ -1628,29 +1701,15 @@ class BaseSpatialDataConverter(BaseMSIConverter, ABC):
             y_coords: NDArray[np.float64] = adata.obs["spatial_y"].values
             half_pixel_um = self.pixel_size_um / 2
 
-            # A volume's footprints carry the depth of the slice they were
-            # acquired on, so they meet the TIC volume at "global" instead
-            # of stacking flat at z=0. spatial_z is already the micrometre
-            # depth (z index * z_spacing_um), so it needs no scaling here
-            # -- the same column the table stores, which is what keeps the
-            # two elements agreeing.
-            z_coords: Optional[NDArray[np.float64]] = None
-            if is_3d and self._is_volume:
-                z_coords = adata.obs["spatial_z"].values
-
+            # Footprints are flat, on every route including volumes. A
+            # slice's depth lives on the TIC image's Scale and in
+            # obs["spatial_z"]; see the docstring for why it is not also
+            # put on the geometry.
             for i in range(len(adata)):
                 x, y = x_coords[i], y_coords[i]
                 x0, y0 = x - half_pixel_um, y - half_pixel_um
                 x1, y1 = x + half_pixel_um, y + half_pixel_um
-
-                if z_coords is None:
-                    pixel_box = box(x0, y0, x1, y1)
-                else:
-                    z = float(z_coords[i])
-                    pixel_box = Polygon(
-                        [(x0, y0, z), (x1, y0, z), (x1, y1, z), (x0, y1, z)]
-                    )
-                geometries.append(pixel_box)
+                geometries.append(box(x0, y0, x1, y1))
 
         # Create GeoDataFrame with appropriate index
         if valid_indices is not None:

{thyra-3.2.1 → thyra-3.3.0}/thyra/converters/spatialdata/spatialdata_2d_converter.py

@@ -328,9 +328,7 @@ class SpatialData2DConverter(BaseSpatialDataConverter):
 
                 # Add to tables and create shapes
                 data_structures["tables"][slice_id] = table
-                data_structures["shapes"][region_key] = self._create_pixel_shapes(
-                    adata, is_3d=False
-                )
+                data_structures["shapes"][region_key] = self._create_pixel_shapes(adata)
 
                 # Create TIC image for this slice
                 tic_values = slice_data["tic_values"]

{thyra-3.2.1 → thyra-3.3.0}/thyra/converters/spatialdata/spatialdata_3d_converter.py

@@ -246,9 +246,7 @@ class SpatialData3DConverter(BaseSpatialDataConverter):
 
             # Add to tables and create shapes
             data_structures["tables"][self.dataset_id] = table
-            data_structures["shapes"][region_key] = self._create_pixel_shapes(
-                adata, is_3d=True
-            )
+            data_structures["shapes"][region_key] = self._create_pixel_shapes(adata)
 
             # Create TIC image
             self._create_tic_image(data_structures)

{thyra-3.2.1 → thyra-3.3.0}/thyra/converters/spatialdata/streaming_converter.py

@@ -1004,9 +1004,7 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
 
                 # Add to tables and create shapes
                 data_structures["tables"][slice_id] = table
-                data_structures["shapes"][region_key] = self._create_pixel_shapes(
-                    adata, is_3d=False
-                )
+                data_structures["shapes"][region_key] = self._create_pixel_shapes(adata)
 
                 # Create TIC image for this slice
                 tic_values = slice_data["tic_values"]
@@ -1702,7 +1700,10 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
         var_group.attrs["encoding-type"] = "dataframe"
         var_group.attrs["encoding-version"] = "0.2.0"
         var_group.attrs["_index"] = "_index"
-        var_group.attrs["column-order"] = ["mz"]
+        # A reader whose native axis is not m/z keeps that axis here too, so
+        # this route stores the same columns as the dataframe-based ones.
+        annotations = self._validated_mass_axis_annotations()
+        var_group.attrs["column-order"] = ["mz"] + sorted(annotations)
 
         mz_values = self._common_mass_axis
         if mz_values is None:
@@ -1725,6 +1726,11 @@ class StreamingSpatialDataConverter(BaseSpatialDataConverter):
         a.attrs["encoding-type"] = "array"
         a.attrs["encoding-version"] = "0.2.0"
 
+        for name in sorted(annotations):
+            a = var_group.create_array(name, data=np.asarray(annotations[name]))
+            a.attrs["encoding-type"] = "array"
+            a.attrs["encoding-version"] = "0.2.0"
+
         # uns (metadata)
         uns_group = table_group.create_group("uns")
         uns_group.attrs["encoding-type"] = "dict"

{thyra-3.2.1 → thyra-3.3.0}/thyra/core/base_reader.py

@@ -213,6 +213,30 @@ class BaseMSIReader(ABC):
         # Return only the valid indices and their corresponding intensities
         return indices[indices_valid], intensities[indices_valid]
 
+    def get_mass_axis_annotations(
+        self,
+    ) -> Optional[dict]:
+        """Get extra per-channel columns to store alongside the m/z axis.
+
+        Returns a mapping of column name to an array with one entry per
+        entry of :meth:`get_common_mass_axis`. These are written into the
+        table's ``var`` next to ``mz``.
+
+        This exists so a format whose native axis is not m/z can keep that
+        axis in the output. Time-of-flight instruments measure flight time
+        and derive m/z from a calibration, so storing the flight time makes
+        the conversion reversible without the reader: a later recalibration
+        can be applied to the stored times directly.
+
+        Annotations are dropped if their length does not match the axis the
+        converter actually writes, which is what happens when resampling is
+        enabled and the axis is rebuilt.
+
+        Returns:
+            Mapping of column name to per-channel values, or None.
+        """
+        return None
+
     def get_region_map(self) -> Optional[dict]:
         """Get per-pixel region mapping for multi-region datasets.
 

{thyra-3.2.1 → thyra-3.3.0}/thyra/core/registry.py

@@ -36,11 +36,12 @@ class MSIRegistry:
         self._lock = RLock()
         self._readers: Dict[str, Type[BaseMSIReader]] = {}
         self._converters: Dict[str, Type[BaseMSIConverter]] = {}
-        # Extension mapping for file-based formats
+        # Extension mapping for file-based formats. '.raw' is deliberately
+        # absent: it is claimed by both Waters and PHI and can only be
+        # resolved by inspecting the path (see _detect_raw_format).
         self._extension_to_format = {
             ".imzml": "imzml",
             ".d": "bruker",
-            ".raw": "waters",
         }
 
     def register_reader(
@@ -112,6 +113,25 @@ class MSIRegistry:
             pass
         return False
 
+    def _detect_phi_format(self, path: Path) -> bool:
+        """Check if path is a PHI SmartSoft-TOF .raw file.
+
+        PHI writes a single file whose ASCII header opens with the four-byte
+        ``SOFH`` magic. This is what distinguishes it from Waters .raw data,
+        which is a directory.
+
+        Args:
+            path: File path to check
+
+        Returns:
+            True if the file starts with the PHI SOFH magic
+        """
+        try:
+            with path.open("rb") as handle:
+                return handle.read(4) == b"SOFH"
+        except (OSError, PermissionError):
+            return False
+
     def detect_format(self, input_path: Path) -> str:
         """Detect MSI format from input path.
 
@@ -120,6 +140,7 @@ class MSIRegistry:
         - .d directories (Bruker timsTOF)
         - Folders with .dat + _poslog.txt (Bruker Rapiflex)
         - .raw directories (Waters MassLynx)
+        - .raw files (PHI SmartSoft-TOF ToF-SIMS)
         """
         if not input_path.exists():
             raise ValueError(f"Input path does not exist: {input_path}")
@@ -136,7 +157,7 @@ class MSIRegistry:
         if extension == ".d":
             return self._detect_bruker_d_format(input_path)
         if extension == ".raw":
-            return self._detect_waters_raw_format(input_path)
+            return self._detect_raw_format(input_path)
         if input_path.is_dir():
             return self._detect_directory_format(input_path)
         self._raise_unsupported_format(input_path)
@@ -152,16 +173,25 @@ class MSIRegistry:
             return bruker_format
         raise ValueError("Bruker .d directory missing analysis " f"files: {input_path}")
 
-    def _detect_waters_raw_format(self, input_path: Path) -> str:
-        """Validate and detect Waters format from .raw directory."""
-        if not input_path.is_dir():
+    def _detect_raw_format(self, input_path: Path) -> str:
+        """Resolve a .raw path to the vendor that wrote it.
+
+        Two vendors claim the extension and they are distinguished by shape:
+        Waters .raw is a directory of _FUNC*.DAT files, PHI .raw is a single
+        file whose header begins with the SOFH magic.
+        """
+        if input_path.is_dir():
+            if self._detect_waters_format(input_path):
+                return "waters"
             raise ValueError(
-                "Waters format requires .raw directory, " f"got file: {input_path}"
+                "Waters .raw directory missing " f"_FUNC*.DAT files: {input_path}"
             )
-        if self._detect_waters_format(input_path):
-            return "waters"
+        if self._detect_phi_format(input_path):
+            return "phi"
         raise ValueError(
-            "Waters .raw directory missing " f"_FUNC*.DAT files: {input_path}"
+            f"Unrecognised .raw file: {input_path}. Expected either a Waters "
+            "directory containing _FUNC*.DAT files, or a PHI SmartSoft-TOF "
+            "file beginning with the SOFH magic."
         )
 
     def _detect_directory_format(self, input_path: Path) -> str:
@@ -179,7 +209,8 @@ class MSIRegistry:
             ".imzml",
             ".d (timsTOF)",
             "folder (Rapiflex)",
-            ".raw (Waters)",
+            ".raw directory (Waters)",
+            ".raw file (PHI SmartSoft-TOF)",
         ]
         raise ValueError(
             f"Unsupported format for '{input_path}'. "
@@ -265,8 +296,7 @@ def detect_format(input_path: Path) -> str:
         input_path: Path to MSI data file or directory
 
     Returns:
-        Format name ('imzml', 'bruker', 'rapiflex',
-        or 'waters')
+        Format name ('imzml', 'bruker', 'rapiflex', 'waters', or 'phi')
     """
     return _registry.detect_format(input_path)
 

{thyra-3.2.1 → thyra-3.3.0}/thyra/metadata/extractors/__init__.py

@@ -30,12 +30,14 @@ Example usage:
 
 from .bruker_extractor import BrukerMetadataExtractor
 from .imzml_extractor import ImzMLMetadataExtractor
+from .phi_extractor import PhiMetadataExtractor
 from .waters_extractor import WatersMetadataExtractor
 
 # Public API
 __all__ = [
     "BrukerMetadataExtractor",
     "ImzMLMetadataExtractor",
+    "PhiMetadataExtractor",
     "WatersMetadataExtractor",
 ]
 
@@ -46,6 +48,7 @@ FORMAT_EXTRACTORS = {
     "tsf": BrukerMetadataExtractor,
     "tdf": BrukerMetadataExtractor,
     "waters": WatersMetadataExtractor,
+    "phi": PhiMetadataExtractor,
 }