chromparser 0.1.0__py3-none-any.whl

chromparser/model.py

@@ -0,0 +1,254 @@
+"""Data model returned by all readers: `Injection`, `Signal` and `MSData`.
+
+See `docs/model.md` for the reader contract. Metadata values are JSON types
+(str, int, float, bool, lists and dicts of them); timestamps in metadata are
+ISO 8601 strings. Unknown values are left out rather than stored as `None` or
+`""`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, fields
+from datetime import datetime
+from typing import Any
+
+import numpy as np
+
+SPECTRUM_TYPES = ("centroid", "profile")
+
+#: Bit of `MSData.flags` set for peaks recorded with a saturated detector.
+SATURATED = 1
+
+
+def _equal(a: Any, b: Any) -> bool:
+    """Compare values; arrays must agree in dtype, shape and content."""
+    if isinstance(a, np.ndarray) or isinstance(b, np.ndarray):
+        if not (isinstance(a, np.ndarray) and isinstance(b, np.ndarray)):
+            return False
+        return (
+            a.dtype == b.dtype
+            and a.shape == b.shape
+            and bool(np.array_equal(a, b, equal_nan=a.dtype.kind in "fc"))
+        )
+    if isinstance(a, dict) and isinstance(b, dict):
+        return a.keys() == b.keys() and all(_equal(a[k], b[k]) for k in a)
+    if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)):
+        return len(a) == len(b) and all(map(_equal, a, b))
+    return bool(a == b)
+
+
+class _ArrayEq:
+    """Dataclass mixin: `==` compares all fields, arrays with `_equal`."""
+
+    def __eq__(self, other: object) -> bool:
+        if type(other) is not type(self):
+            return NotImplemented
+        return all(
+            _equal(getattr(self, f.name), getattr(other, f.name))
+            for f in fields(self)  # type: ignore[arg-type]
+        )
+
+    __hash__ = None  # type: ignore[assignment]
+
+
+@dataclass(eq=False)
+class Signal(_ArrayEq):
+    """One detector trace of one injection (FID, TCD, single-wavelength UV, ...).
+
+    Attributes
+    ----------
+    channel : str
+        Channel name, e.g. ``"FID_left"``.
+    time : numpy.ndarray
+        Retention time in minutes (float64).
+    values : numpy.ndarray
+        Signal values in `unit`, with the dtype the file stores.
+    unit : str or None
+        Signal unit, e.g. ``"pA"``; `None` when the file does not say.
+    acquisition_start : datetime or None
+        Start of the data acquisition (timezone-aware, UTC).
+    metadata : dict
+        Further metadata, e.g. ``detector``, ``driver``.
+    sampling_interval : float or None
+        Time between two points in seconds, for regularly sampled signals.
+    """
+
+    channel: str
+    time: np.ndarray
+    values: np.ndarray
+    unit: str | None = None
+    acquisition_start: datetime | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    sampling_interval: float | None = None
+
+    def __len__(self) -> int:
+        return len(self.time)
+
+
+@dataclass(eq=False)
+class MSData(_ArrayEq):
+    """Mass spectra of one injection, stored back to back (CSR layout).
+
+    The peaks of the scan at index ``i`` are
+    ``mz[peak_offset[i]:peak_offset[i + 1]]`` (likewise `intensity` and
+    `flags`). Use `spectrum` for one scan and `xic` for extracted-ion
+    chromatograms.
+
+    Attributes
+    ----------
+    name : str
+        Name of the MS data, e.g. ``"MSDevice"``.
+    scan_number : numpy.ndarray
+        Scan numbers (int32) as the file stores them (Thermo counts from 1,
+        Shimadzu from 0).
+    retention_time : numpy.ndarray
+        Retention time of each scan in minutes (float64).
+    tic : numpy.ndarray
+        Total ion current of each scan (float64).
+    base_peak_mz, base_peak_intensity : numpy.ndarray or None
+        m/z and intensity of the most intense peak of each scan; `None` when
+        the file has no such field. When computed by the reader, their names
+        are listed in ``metadata["computed"]``.
+    filters : list of str or None
+        Scan filter of each scan, e.g. ``"+ c EI Full ms [33.00-500.00]"``.
+    mz : numpy.ndarray
+        m/z of all peaks, as stored (float32 for Thermo, float64 for Shimadzu).
+    intensity : numpy.ndarray
+        Intensity of all peaks, as stored (float32 for Thermo, uint32 for
+        Shimadzu).
+    peak_offset : numpy.ndarray
+        Start of each scan's peaks, plus the total number of peaks (int64,
+        one more entry than scans).
+    acquisition_start : datetime or None
+        Start of the MS acquisition (timezone-aware, UTC).
+    metadata : dict
+        Further metadata, e.g. ``instrument_model``, ``ionisation``.
+    ms_level : numpy.ndarray or None
+        MS level of each scan (uint8).
+    flags : numpy.ndarray or None
+        Per-peak flags (uint8); bit 0 (`SATURATED`) marks a saturated detector.
+    spectrum_type : str
+        ``"centroid"`` or ``"profile"``.
+    """
+
+    name: str
+    scan_number: np.ndarray
+    retention_time: np.ndarray
+    tic: np.ndarray
+    base_peak_mz: np.ndarray | None
+    base_peak_intensity: np.ndarray | None
+    filters: list[str] | None
+    mz: np.ndarray
+    intensity: np.ndarray
+    peak_offset: np.ndarray
+    acquisition_start: datetime | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    ms_level: np.ndarray | None = None
+    flags: np.ndarray | None = None
+    spectrum_type: str = "centroid"
+
+    def __post_init__(self) -> None:
+        n = len(self.scan_number)
+        per_scan = {
+            "retention_time": self.retention_time,
+            "tic": self.tic,
+            "base_peak_mz": self.base_peak_mz,
+            "base_peak_intensity": self.base_peak_intensity,
+            "filters": self.filters,
+            "ms_level": self.ms_level,
+        }
+        for name, values in per_scan.items():
+            if values is not None and len(values) != n:
+                raise ValueError(f"MSData.{name} has {len(values)} entries, not {n}.")
+        if len(self.peak_offset) != n + 1:
+            raise ValueError("MSData.peak_offset must have one entry more than scans.")
+        n_peaks = int(self.peak_offset[-1])
+        for name in ("mz", "intensity", "flags"):
+            values = getattr(self, name)
+            if values is not None and len(values) != n_peaks:
+                raise ValueError(
+                    f"MSData.{name} has {len(values)} peaks, peak_offset says {n_peaks}."
+                )
+        if self.spectrum_type not in SPECTRUM_TYPES:
+            raise ValueError(f"Unknown spectrum type {self.spectrum_type!r}.")
+
+    def __len__(self) -> int:
+        return len(self.scan_number)
+
+    def spectrum(self, scan_number: int) -> tuple[np.ndarray, np.ndarray]:
+        """Return the m/z and intensity arrays of one scan.
+
+        Parameters
+        ----------
+        scan_number : int
+            Scan number as stored in `scan_number`.
+
+        Returns
+        -------
+        mz, intensity : numpy.ndarray
+            Views into `mz` and `intensity`.
+
+        Raises
+        ------
+        KeyError
+            If there is no scan with this number.
+        """
+        index = int(np.searchsorted(self.scan_number, scan_number))
+        if index == len(self) or self.scan_number[index] != scan_number:
+            raise KeyError(f"No scan {scan_number}.")
+        peaks = slice(self.peak_offset[index], self.peak_offset[index + 1])
+        return self.mz[peaks], self.intensity[peaks]
+
+    def xic(self, mz: float, tolerance: float = 0.5) -> np.ndarray:
+        """Extracted-ion chromatogram.
+
+        Sums the intensity of each scan within
+        ``mz - tolerance <= m/z <= mz + tolerance``.
+
+        Parameters
+        ----------
+        mz : float
+            Centre of the m/z window.
+        tolerance : float
+            Half width of the window.
+
+        Returns
+        -------
+        numpy.ndarray
+            One intensity (float64) per scan, to plot against `retention_time`.
+        """
+        selected = (self.mz >= mz - tolerance) & (self.mz <= mz + tolerance)
+        scan_index = np.repeat(np.arange(len(self)), np.diff(self.peak_offset))
+        return np.bincount(
+            scan_index[selected],
+            weights=self.intensity[selected].astype(np.float64),
+            minlength=len(self),
+        )
+
+
+@dataclass(eq=False)
+class Injection(_ArrayEq):
+    """One injection with its signals and MS data.
+
+    Attributes
+    ----------
+    name : str
+        Injection name (``""`` when unnamed).
+    inject_time : datetime or None
+        Injection time as local time of the instrument: timezone-aware when the
+        file stores the UTC offset, naive when it stores only the wall time
+        (Agilent ``.ch``, some Chromeleon text exports), `None` when unknown.
+    signals : dict of str to Signal
+        Signals by channel name.
+    ms : dict of str to MSData
+        MS data by name.
+    metadata : dict
+        Further metadata, e.g. ``sample_name``, ``sequence``,
+        ``instrument_method``.
+    """
+
+    name: str
+    inject_time: datetime | None = None
+    signals: dict[str, Signal] = field(default_factory=dict)
+    ms: dict[str, MSData] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)

chromparser/sniff.py

@@ -0,0 +1,80 @@
+"""Detect the format of a file from its content.
+
+Magic bytes, OLE2 storages, zip members and the content of `.D` folders
+decide; the file extension is only used in error messages.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ._util import head
+from .errors import UnsupportedFormat
+
+HDF5 = "hdf5"
+
+_HINTS = {
+    ".cmbx": "chromeleon-cmbx",
+    ".dx": "agilent-dx",
+    ".raw": "chromeleon-raw or thermo-raw",
+    ".qgd": "shimadzu-qgd",
+    ".gcd": "shimadzu-gcd",
+    ".d": "agilent-d",
+    ".ch": "agilent-ch",
+    ".txt": "chromeleon-txt",
+    ".cdf": "andi-ms or andi-chrom",
+    ".h5": HDF5,
+    ".hdf5": HDF5,
+}
+
+_RECOGNISED_UNSUPPORTED = {
+    b"CpAu": "a Chromeleon audit trail (CpAu), which holds no chromatogram",
+    b"CDF\x01": "a netCDF file without ANDI MS or chromatography data",
+    b"CDF\x02": "a netCDF file without ANDI MS or chromatography data",
+    b"CDF\x05": "a CDF-5 netCDF file, which chromparser does not read",
+}
+
+
+def sniff(path: str | Path) -> str:
+    """Return the format id of a file or folder.
+
+    Parameters
+    ----------
+    path : str or Path
+        File or folder (Agilent `.D`).
+
+    Returns
+    -------
+    str
+        One of the ids in `chromparser.vendors.FORMATS` (e.g.
+        ``"shimadzu-qgd"``), or ``"hdf5"`` for files written by chromparser
+        or cmbx-reader.
+
+    Raises
+    ------
+    FileNotFoundError
+        If `path` does not exist.
+    UnsupportedFormat
+        If the content is not a supported format.
+    """
+    from .hdf5 import sniff_hdf5
+    from .vendors import FORMATS
+
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(f"File not found: {path}")
+    if sniff_hdf5(path):
+        return HDF5
+    for fmt in FORMATS:
+        if fmt.sniff(path):
+            return fmt.id
+    if path.is_file():
+        start = head(path, 8)
+        for magic, what in _RECOGNISED_UNSUPPORTED.items():
+            if start.startswith(magic):
+                raise UnsupportedFormat(f"{path.name} is {what}.")
+    hint = _HINTS.get(path.suffix.lower())
+    detail = f" (the extension suggests {hint})" if hint else ""
+    raise UnsupportedFormat(
+        f"{path.name} is not a chromatography file chromparser can read{detail}."
+    )

chromparser/vendors/__init__.py

@@ -0,0 +1,55 @@
+"""Readers of the vendor formats and the registry used by `chromparser.read`."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+from ..model import Injection
+from .agilent import (
+    read_agilent_ch,
+    read_agilent_d,
+    read_agilent_dx,
+    sniff_agilent_ch,
+    sniff_agilent_d,
+    sniff_agilent_dx,
+)
+from .andi import read_andi_chrom, read_andi_ms, sniff_andi_chrom, sniff_andi_ms
+from .chromeleon import (
+    read_chromeleon_raw,
+    read_chromeleon_txt,
+    read_cmbx,
+    sniff_chromeleon_raw,
+    sniff_chromeleon_txt,
+    sniff_cmbx,
+)
+from .shimadzu import read_gcd, read_qgd, sniff_gcd, sniff_qgd
+from .thermo import read_thermo_raw, sniff_thermo_raw
+
+
+@dataclass(frozen=True)
+class Format:
+    """A file format: id, description, content sniffer and reader."""
+
+    id: str
+    description: str
+    sniff: Callable[[str | Path], bool]
+    read: Callable[..., list[Injection]]
+
+
+#: Vendor formats in the order they are tried by `sniff`: formats with a
+#: specific signature first, the heuristic ones (Agilent `.ch`, text) last.
+FORMATS = [
+    Format("chromeleon-cmbx", "Chromeleon 7 archive (.cmbx)", sniff_cmbx, read_cmbx),
+    Format("agilent-dx", "Agilent OpenLab archive (.dx)", sniff_agilent_dx, read_agilent_dx),
+    Format("chromeleon-raw", "Chromeleon 7 raw signal file (.raw)", sniff_chromeleon_raw, read_chromeleon_raw),
+    Format("thermo-raw", "Thermo Finnigan MS file (.raw)", sniff_thermo_raw, read_thermo_raw),
+    Format("shimadzu-qgd", "Shimadzu GCMSsolution GC-MS file (.qgd)", sniff_qgd, read_qgd),
+    Format("shimadzu-gcd", "Shimadzu GC file (.gcd)", sniff_gcd, read_gcd),
+    Format("andi-ms", "ANDI-MS netCDF GC-MS file (.cdf)", sniff_andi_ms, read_andi_ms),
+    Format("andi-chrom", "ANDI chromatography (AIA) netCDF file (.cdf)", sniff_andi_chrom, read_andi_chrom),
+    Format("agilent-d", "Agilent data folder (.D)", sniff_agilent_d, read_agilent_d),
+    Format("agilent-ch", "Agilent chromatogram (.ch)", sniff_agilent_ch, read_agilent_ch),
+    Format("chromeleon-txt", "Chromeleon text export (.txt)", sniff_chromeleon_txt, read_chromeleon_txt),
+]  # fmt: skip

chromparser/vendors/agilent/__init__.py

@@ -0,0 +1,17 @@
+"""Agilent ChemStation / OpenLab: `.ch` files, `.D` folders and `.dx` archives."""
+
+from __future__ import annotations
+
+from .ch import decode_ch, read_agilent_ch, sniff_agilent_ch
+from .dot_d import read_agilent_d, sniff_agilent_d
+from .dx import read_agilent_dx, sniff_agilent_dx
+
+__all__ = [
+    "decode_ch",
+    "read_agilent_ch",
+    "read_agilent_d",
+    "read_agilent_dx",
+    "sniff_agilent_ch",
+    "sniff_agilent_d",
+    "sniff_agilent_dx",
+]