chromparser 0.1.0__py3-none-any.whl

chromparser/vendors/shimadzu/qgd.py

@@ -0,0 +1,196 @@
+"""Shimadzu GCMSsolution GC-MS files (`.qgd`, storage ``GCMS Raw Data``).
+
+Centroid spectra in full scan mode. Masses are stored in units of 1/20 u and
+returned as float64 m/z; intensities are 2, 3 or 4 bytes per scan and are
+returned as uint32, with bit 31 (saturated detector) moved to `MSData.flags`.
+SIM/MRM scans (format code 0) raise `UnsupportedVariant`. See
+`docs/formats/shimadzu.md` for the layout.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+
+from ...errors import CorruptFile, UnsupportedFormat, UnsupportedVariant, warn
+from ...model import SATURATED, Injection, MSData
+from .ole import QGD_STORAGE, file_properties, open_ole, read_stream, storages
+
+MZ_SCALE = 20  # raw masses are m/z * 20
+INTENSITY_BYTES = {1: 4, 2: 2, 3: 3}  # format code -> bytes per intensity
+SATURATED_BIT = 1 << 31
+SCAN_HEADER = 32
+CHUNK = 4096  # scans decoded at a time
+STREAMS = ("Retention Time", "Spectrum Index", "TIC Data", "MS Raw Data")
+
+
+def _spectrum_index(data: bytes, n_scans: int, name: str) -> np.ndarray:
+    if len(data) == 4 * n_scans:
+        return np.frombuffer(data, "<u4").astype(np.int64)
+    if len(data) == 2 + 8 * n_scans:
+        return np.frombuffer(data, "<u8", offset=2).astype(np.int64)
+    raise CorruptFile(
+        f"{name}: spectrum index ({len(data)} bytes) does not match "
+        f"{n_scans} retention times."
+    )
+
+
+def decode_qgd(streams: dict[str, bytes], name: str = "qgd") -> MSData:
+    """Decode the ``GCMS Raw Data`` streams of a `.qgd` file.
+
+    Parameters
+    ----------
+    streams : dict of str to bytes
+        ``"Retention Time"``, ``"Spectrum Index"``, ``"MS Raw Data"`` and
+        optionally ``"TIC Data"``, which every scan is checked against.
+    name : str
+        Name of the MS data and file name for messages.
+
+    Returns
+    -------
+    MSData
+        Without acquisition start and file metadata (see `read_qgd`).
+
+    Raises
+    ------
+    UnsupportedVariant
+        For SIM/MRM or unknown scan formats.
+    CorruptFile
+        If the streams are inconsistent or truncated.
+    """
+    rt = np.frombuffer(streams["Retention Time"], "<i4")
+    raw = np.frombuffer(streams["MS Raw Data"], np.uint8)
+    starts = _spectrum_index(streams["Spectrum Index"], len(rt), name)
+    n_scans = len(starts)
+    metadata: dict = {}
+    if n_scans and np.any(starts + SCAN_HEADER > len(raw)):
+        raise CorruptFile(f"{name}: spectrum index points past the end of the data.")
+
+    header = raw[starts[:, None] + np.arange(SCAN_HEADER)]
+    scan_no = header[:, 0:4].copy().view("<u4")[:, 0]
+    header_rt = header[:, 4:8].copy().view("<u4")[:, 0]
+    codes = header[:, 20:22].copy().view("<u2")[:, 0]
+    counts = header[:, 22:24].copy().view("<u2")[:, 0].astype(np.int64)
+    if n_scans and not np.array_equal(
+        scan_no.astype(np.int64) - int(scan_no[0]), np.arange(n_scans)
+    ):
+        raise CorruptFile(f"{name}: scan headers are out of order.")
+    unknown = sorted(set(np.unique(codes).tolist()) - set(INTENSITY_BYTES))
+    if unknown:
+        raise UnsupportedVariant(
+            f"{name}: unsupported scan format code(s) {unknown} "
+            "(0 is SIM/MRM, which is not supported)."
+        )
+    if not np.array_equal(header_rt.astype(np.int64), rt.astype(np.int64)):
+        warn(metadata, f"{name}: scan headers and Retention Time disagree.")
+    width_of_code = np.zeros(max(INTENSITY_BYTES) + 1, dtype=np.int64)
+    for code, width in INTENSITY_BYTES.items():
+        width_of_code[code] = width
+    widths = width_of_code[codes]
+    ends = starts + SCAN_HEADER + counts * (2 + widths)
+    if n_scans and ends.max() > len(raw):
+        raise CorruptFile(f"{name}: the MS data is truncated.")
+    if np.any(starts[1:] < ends[:-1]):
+        raise CorruptFile(f"{name}: scans overlap or are out of order.")
+
+    peak_offset = np.zeros(n_scans + 1, dtype=np.int64)
+    np.cumsum(counts, out=peak_offset[1:])
+    mz_raw = np.empty(peak_offset[-1], dtype=np.uint16)
+    intensity = np.empty(peak_offset[-1], dtype=np.uint32)
+    for first in range(0, n_scans, CHUNK):
+        chunk = slice(first, min(first + CHUNK, n_scans))
+        n = counts[chunk]
+        peaks = slice(peak_offset[chunk.start], peak_offset[chunk.stop])
+        within = np.arange(peaks.stop - peaks.start) - np.repeat(
+            peak_offset[chunk] - peaks.start, n
+        )
+        width = np.repeat(widths[chunk], n)
+        pos = np.repeat(starts[chunk] + SCAN_HEADER, n) + within * (2 + width)
+        mz_raw[peaks] = raw[pos] | (raw[pos + 1].astype(np.uint16) << 8)
+        value = np.zeros(len(pos), dtype=np.uint32)
+        for byte in range(4):
+            has = width > byte
+            value[has] |= raw[pos[has] + 2 + byte].astype(np.uint32) << (8 * byte)
+        intensity[peaks] = value
+
+    flags = (intensity >> 31).astype(np.uint8) * np.uint8(SATURATED)
+    intensity &= np.uint32(SATURATED_BIT - 1)
+    computed = np.bincount(
+        np.repeat(np.arange(n_scans), counts),
+        weights=intensity.astype(np.float64),
+        minlength=n_scans,
+    )
+    stored = streams.get("TIC Data")
+    if stored is None:
+        tic = computed
+        metadata["computed"] = ["tic"]
+    else:
+        tic = np.frombuffer(stored, "<i8").astype(np.float64)
+        if len(tic) != n_scans:
+            raise CorruptFile(f"{name}: TIC Data has {len(tic)} scans, not {n_scans}.")
+        bad = int(np.count_nonzero(tic != computed))
+        if bad:
+            warn(metadata, f"{name}: {bad} scans do not add up to the stored TIC.")
+    return MSData(
+        name=name,
+        scan_number=scan_no.astype(np.int32),
+        retention_time=rt.astype(np.float64) / 60000.0,
+        tic=tic,
+        base_peak_mz=None,
+        base_peak_intensity=None,
+        filters=None,
+        mz=mz_raw.astype(np.float64) / MZ_SCALE,
+        intensity=intensity,
+        peak_offset=peak_offset,
+        metadata=metadata,
+        flags=flags,
+        spectrum_type="centroid",
+    )
+
+
+def read_qgd(path: str | Path) -> list[Injection]:
+    """Read a Shimadzu GCMSsolution `.qgd` file.
+
+    Parameters
+    ----------
+    path : str or Path
+        The `.qgd` file.
+
+    Returns
+    -------
+    list of Injection
+        One injection with one MS run named ``"MS"``. `inject_time` is
+        `None`; the acquisition start (UTC) is in `MSData.acquisition_start`.
+
+    Raises
+    ------
+    UnsupportedFormat
+        If the file holds no GC-MS raw data.
+    """
+    path = Path(path)
+    with open_ole(path) as ole:
+        if not ole.exists(f"{QGD_STORAGE}/MS Raw Data"):
+            raise UnsupportedFormat(f"{path.name} contains no GCMS raw data.")
+        streams = {
+            name: read_stream(ole, f"{QGD_STORAGE}/{name}")
+            for name in STREAMS
+            if ole.exists(f"{QGD_STORAGE}/{name}")
+        }
+        start, metadata = file_properties(ole)
+    missing = [n for n in STREAMS[:2] if n not in streams]
+    if missing:
+        raise CorruptFile(f"{path.name}: stream(s) {missing} are missing.")
+    ms = decode_qgd(streams, path.name)
+    ms.name = "MS"
+    ms.acquisition_start = start
+    return [
+        Injection(
+            name=metadata.get("sample_name", ""), ms={"MS": ms}, metadata=metadata
+        )
+    ]
+
+
+def sniff_qgd(path: str | Path) -> bool:
+    """Whether `path` is an OLE2 file with a ``GCMS Raw Data`` storage."""
+    return QGD_STORAGE in storages(path)

chromparser/vendors/thermo/__init__.py

@@ -0,0 +1,7 @@
+"""Thermo Finnigan `.raw` MS files."""
+
+from __future__ import annotations
+
+from .raw import decode_thermo_raw, read_thermo_ms, read_thermo_raw, sniff_thermo_raw
+
+__all__ = ["decode_thermo_raw", "read_thermo_ms", "read_thermo_raw", "sniff_thermo_raw"]

chromparser/vendors/thermo/raw.py

@@ -0,0 +1,377 @@
+"""Thermo `.raw` MS files (Finnigan format), as embedded in `.cmbx` archives.
+
+Only format version 66 with centroid data from a single quadrupole (packet
+type 24, e.g. ISQ GC-MS) is supported; everything else raises
+`UnsupportedVariant`. See `docs/formats/thermo-raw.md` for the binary layout.
+"""
+
+from __future__ import annotations
+
+import struct
+import warnings
+from datetime import datetime, timezone
+from pathlib import Path
+
+import numpy as np
+
+from ..._util import Source, existing_file, head, read_bytes
+from ...errors import (
+    ChromparserWarning,
+    CorruptFile,
+    UnsupportedFormat,
+    UnsupportedVariant,
+    warn,
+)
+from ...model import Injection, MSData
+
+THERMO_RAW_MAGIC = b"\x01\xa1" + "Finnigan".encode("utf-16-le")
+SUPPORTED_VERSIONS = (66,)
+
+_FILE_HEADER_SIZE = 1356
+_VERSION_OFFSET = 36
+
+# RawFileInfo: 44 byte preamble, 764 unknown bytes, then 64-bit addresses
+_RAW_FILE_INFO_ADDRESSES = 808
+_RUN_HEADER_SIZE = 0x1D98
+_RUN_HEADER_ADDRESSES = 0x1CF0
+_SCAN_INDEX_ENTRY = struct.Struct("<IIHHIII6dQII")
+_PACKET_DESCRIPTOR = struct.Struct("<IffdQ20sQ")
+_SCAN_EVENT_PREAMBLE_SIZE = 132
+
+# packet type of centroid data from single quadrupoles (`LowResolutionSpectrumType4`)
+_CENTROID_PACKET = 24
+
+# scan event preamble bytes (byte index: {value: meaning})
+_POLARITY = 4, {0: "-", 1: "+"}
+_SCAN_DATA = 5, {0: "c", 1: "p"}
+_MS_ORDER = 6, {1: "ms"}
+_SCAN_TYPE = 7, {0: "Full"}
+_IONIZATION = 11, {0: "EI", 1: "CI"}
+
+
+class _Reader:
+    """Bounds-checked little-endian reads from a buffer."""
+
+    def __init__(self, data: bytes):
+        self.data = data
+
+    def unpack(self, fmt: str | struct.Struct, offset: int) -> tuple:
+        layout = fmt if isinstance(fmt, struct.Struct) else struct.Struct("<" + fmt)
+        if offset < 0 or offset + layout.size > len(self.data):
+            raise CorruptFile(
+                f"Truncated Thermo raw file: cannot read {layout.size} bytes "
+                f"at offset {offset}."
+            )
+        return layout.unpack_from(self.data, offset)
+
+    def string(self, offset: int) -> tuple[str, int]:
+        """Read a UTF-16 string preceded by its length in characters (uint32)."""
+        (length,) = self.unpack("I", offset)
+        end = offset + 4 + 2 * length
+        if end > len(self.data):
+            raise CorruptFile(f"Truncated Thermo raw file: string at offset {offset}.")
+        return self.data[offset + 4 : end].decode("utf-16-le"), end
+
+    def skip_strings(self, offset: int, count: int) -> int:
+        for _ in range(count):
+            _, offset = self.string(offset)
+        return offset
+
+
+def _run_header_address(reader: _Reader) -> tuple[int, int, datetime | None]:
+    """Walk the file header to RawFileInfo.
+
+    Returns
+    -------
+    data_address, run_header : int
+        Addresses of the scan data and the run header.
+    acquisition_start : datetime or None
+        In UTC.
+    """
+    pos = _FILE_HEADER_SIZE
+    # SeqRow: 64 bytes injection data, 13 strings, 3 strings + uint32, 15 strings
+    pos = reader.skip_strings(pos + 64, 13)
+    pos = reader.skip_strings(pos, 3) + 4
+    pos = reader.skip_strings(pos, 15)
+    # ASInfo: 24 bytes, 1 string
+    _, pos = reader.string(pos + 24)
+    # RawFileInfo
+    year, month, _, day, hour, minute, second, ms = reader.unpack("8H", pos + 4)
+    (controllers,) = reader.unpack("I", pos + 28)
+    if controllers != 1:
+        raise UnsupportedVariant(
+            f"Thermo raw file with {controllers} instruments: only files with one "
+            "MS instrument are supported."
+        )
+    data_address, _, run_header = reader.unpack("3Q", pos + _RAW_FILE_INFO_ADDRESSES)
+    try:
+        start = datetime(
+            year, month, day, hour, minute, second, ms * 1000, timezone.utc
+        )
+    except ValueError:
+        start = None
+    return data_address, run_header, start
+
+
+def _scan_event(reader: _Reader, error_log: int) -> tuple[bytes, list[tuple]]:
+    """Read the scan event that follows the error log.
+
+    Returns
+    -------
+    preamble : bytes
+        The preamble bytes of the event.
+    ranges : list of tuple
+        The mass ranges of the event.
+    """
+    (errors,) = reader.unpack("I", error_log)
+    pos = error_log + 4
+    for _ in range(errors):  # float32 retention time, message
+        _, pos = reader.string(pos + 4)
+    segments, events = reader.unpack("II", pos)
+    if (segments, events) != (1, 1):
+        raise UnsupportedVariant(
+            f"Thermo raw file with {segments} scan segment(s) and {events} scan "
+            "event(s) in the first segment: only one full scan event is supported."
+        )
+    pos += 8
+    preamble = reader.data[pos : pos + _SCAN_EVENT_PREAMBLE_SIZE]
+    pos += _SCAN_EVENT_PREAMBLE_SIZE
+    precursors, _, n_ranges = reader.unpack("III", pos)
+    if precursors:
+        raise UnsupportedVariant("Thermo raw file with MSn scans is not supported.")
+    ranges = [reader.unpack("dd", pos + 12 + 16 * i) for i in range(n_ranges)]
+    return preamble, ranges
+
+
+def _scan_filter(preamble: bytes, ranges: list[tuple]) -> tuple[str, dict]:
+    """Build the scan filter, e.g. ``+ c EI Full ms [33.00-500.00]``.
+
+    Returns
+    -------
+    filter : str
+        The scan filter.
+    metadata : dict
+        ``polarity``, ``ionisation`` and the mass range.
+    """
+    parts = []
+    for (index, meanings), what in [
+        (_POLARITY, "polarity"),
+        (_SCAN_DATA, "scan data type"),
+        (_IONIZATION, "ionization"),
+        (_SCAN_TYPE, "scan type"),
+        (_MS_ORDER, "MS order"),
+    ]:
+        value = preamble[index]
+        if value not in meanings:
+            raise UnsupportedVariant(
+                f"Thermo raw file with unsupported {what} ({value})."
+            )
+        parts.append(meanings[value])
+    if parts[1] != "c":
+        raise UnsupportedVariant("Thermo raw file with profile data is not supported.")
+    if not ranges:
+        raise UnsupportedVariant(
+            "Thermo raw file without a mass range in its scan event."
+        )
+    masses = ", ".join(f"{low:.2f}-{high:.2f}" for low, high in ranges)
+    metadata = {
+        "polarity": "positive" if parts[0] == "+" else "negative",
+        "ionisation": parts[2],
+        "mass_range_min": min(low for low, _ in ranges),
+        "mass_range_max": max(high for _, high in ranges),
+    }
+    return " ".join(parts) + f" [{masses}]", metadata
+
+
+def _instrument(reader: _Reader, offset: int) -> dict:
+    """Instrument id after the run header: 12 bytes, then strings."""
+    metadata = {}
+    try:
+        pos = offset + 12
+        for key in ["instrument", "instrument_model", "serial_number"]:
+            metadata[key], pos = reader.string(pos)
+        metadata["instrument_software_version"], pos = reader.string(pos)
+    except (CorruptFile, UnicodeDecodeError) as e:
+        warnings.warn(
+            f"Could not read the instrument id: {e}", ChromparserWarning, stacklevel=4
+        )
+    return {key: value for key, value in metadata.items() if value}
+
+
+def decode_thermo_raw(data: bytes, name: str = "MS") -> MSData:
+    """Decode the bytes of a Thermo `.raw` file.
+
+    Parameters
+    ----------
+    data : bytes
+        Content of the file.
+    name : str
+        Name of the MS data.
+
+    Returns
+    -------
+    MSData
+        Centroid spectra; m/z and intensity are float32 as stored.
+
+    Raises
+    ------
+    UnsupportedFormat
+        If the data is not a Thermo raw file.
+    UnsupportedVariant
+        If the file is not a supported variant (see the module docstring).
+    CorruptFile
+        If the file is truncated or inconsistent.
+    """
+    if not data.startswith(THERMO_RAW_MAGIC):
+        raise UnsupportedFormat("Not a Thermo raw file (missing 'Finnigan' header).")
+    reader = _Reader(data)
+    (version,) = reader.unpack("I", _VERSION_OFFSET)
+    if version not in SUPPORTED_VERSIONS:
+        raise UnsupportedVariant(
+            f"Thermo raw file format version {version} is not supported "
+            f"(supported: {', '.join(map(str, SUPPORTED_VERSIONS))})."
+        )
+
+    data_address, run_header, acquisition_start = _run_header_address(reader)
+    first, last = reader.unpack("II", run_header + 8)
+    (index, data_check, _, error_log, _, trailer, _, _, run_header_check) = (
+        reader.unpack("9Q", run_header + _RUN_HEADER_ADDRESSES)
+    )
+    if (data_check, run_header_check) != (data_address, run_header):
+        raise CorruptFile("Thermo raw file with inconsistent run header addresses.")
+    n_scans = last - first + 1
+    if first < 1 or n_scans < 0 or index + n_scans * _SCAN_INDEX_ENTRY.size > trailer:
+        raise CorruptFile(
+            f"Thermo raw file with an invalid scan range {first}-{last} or scan index."
+        )
+
+    preamble, ranges = _scan_event(reader, error_log)
+    scan_filter, event_metadata = _scan_filter(preamble, ranges)
+
+    metadata: dict = {"format_version": version}
+    scans = np.empty(n_scans, dtype=np.int32)
+    stats = np.empty((n_scans, 6))
+    counts = np.empty(n_scans, dtype=np.int64)
+    chunks = []
+    peaks_end = 0  # packets are contiguous: peaks, peak flags, descriptor
+    flagged = []
+    for i in range(n_scans):
+        entry = reader.unpack(_SCAN_INDEX_ENTRY, index + i * _SCAN_INDEX_ENTRY.size)
+        scan, packet_type, *_ = entry[4:7]
+        if scan != first + i:
+            raise CorruptFile(f"Thermo raw file: scan {scan} found at index {i}.")
+        if packet_type != _CENTROID_PACKET:
+            raise UnsupportedVariant(
+                f"Thermo raw file: scan {scan} has packet type {packet_type}. Only "
+                f"centroid data from single quadrupoles (type {_CENTROID_PACKET}) "
+                "is supported."
+            )
+        if entry[2:4] != (0, 0):
+            raise UnsupportedVariant(
+                f"Thermo raw file: scan {scan} has an unknown scan event."
+            )
+        stats[i] = entry[7:13]
+        scans[i] = scan
+
+        descriptor = entry[13]
+        _, _, _, _, offset, reserved, size = reader.unpack(
+            _PACKET_DESCRIPTOR, data_address + descriptor
+        )
+        count = size // 8
+        if (
+            reserved.strip(b"\0")
+            or size % 8
+            or offset != peaks_end
+            or offset + size + count != descriptor
+        ):
+            raise UnsupportedVariant(
+                f"Thermo raw file: unsupported data packet of scan {scan}."
+            )
+        peaks_end = descriptor + _PACKET_DESCRIPTOR.size
+        start = data_address + offset
+        chunks.append(data[start : start + size])
+        counts[i] = count
+        if data[start + size : start + size + count].strip(b"\0"):
+            flagged.append(int(scan))
+    if data_address + peaks_end != run_header:
+        raise CorruptFile("Thermo raw file: unexpected data between the scans.")
+    if flagged:
+        warn(
+            metadata,
+            f"Thermo raw file: {len(flagged)} scan(s) have peak flags of unknown "
+            f"meaning, first scan {flagged[0]}; the flags are not read.",
+        )
+
+    peaks = np.frombuffer(b"".join(chunks), dtype="<f4").reshape(-1, 2)
+    peak_offset = np.zeros(n_scans + 1, dtype=np.int64)
+    np.cumsum(counts, out=peak_offset[1:])
+
+    metadata.update(event_metadata)
+    metadata.update(_instrument(reader, run_header + _RUN_HEADER_SIZE))
+    return MSData(
+        name=name,
+        scan_number=scans,
+        retention_time=stats[:, 0].copy(),
+        tic=stats[:, 1].copy(),
+        base_peak_mz=stats[:, 3].copy(),
+        base_peak_intensity=stats[:, 2].copy(),
+        filters=[scan_filter] * n_scans,
+        mz=peaks[:, 0].astype(np.float32),
+        intensity=peaks[:, 1].astype(np.float32),
+        peak_offset=peak_offset,
+        acquisition_start=acquisition_start,
+        metadata=metadata,
+        ms_level=np.ones(n_scans, dtype=np.uint8),
+        spectrum_type="centroid",
+    )
+
+
+def read_thermo_ms(source: Source, name: str = "MS") -> MSData:
+    """Read a Thermo `.raw` MS file into `MSData`.
+
+    This is cmbx-reader's ``read_thermo_raw``.
+
+    Parameters
+    ----------
+    source : str, Path, bytes or file object
+        The Thermo raw file.
+    name : str
+        Name of the MS data.
+
+    Returns
+    -------
+    MSData
+        With `acquisition_start` in UTC.
+    """
+    return decode_thermo_raw(read_bytes(source), name)
+
+
+def read_thermo_raw(path: str | Path, *, name: str = "MS") -> list[Injection]:
+    """Read a Thermo `.raw` MS file, e.g. one extracted from a `.cmbx` archive.
+
+    Parameters
+    ----------
+    path : str or Path
+        The Thermo raw file.
+    name : str
+        Name of the MS data.
+
+    Returns
+    -------
+    list of Injection
+        One injection holding the MS data.
+    """
+    ms = read_thermo_ms(existing_file(path), name)
+    metadata = {
+        key: ms.metadata[key]
+        for key in ("instrument", "instrument_model")
+        if key in ms.metadata
+    }
+    return [Injection(name="", ms={name: ms}, metadata=metadata)]
+
+
+def sniff_thermo_raw(path: str | Path) -> bool:
+    """Whether `path` is a Thermo raw file (starts with ``01 A1 "Finnigan"``)."""
+    return (
+        Path(path).is_file() and head(path, len(THERMO_RAW_MAGIC)) == THERMO_RAW_MAGIC
+    )