chromparser 0.1.0__py3-none-any.whl

chromparser/cli.py

@@ -0,0 +1,192 @@
+"""Command line interface: ``chromparser convert | info | extract-ms``."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+import numpy as np
+
+from . import __version__, convert, read, sniff, write_andi
+from .errors import ChromparserError
+from .model import Injection, MSData, Signal
+from .vendors.chromeleon import extract_ms_raw
+
+
+def _interval(signal: Signal) -> str:
+    if signal.sampling_interval is None:
+        return "irregular" if len(signal) > 1 else ""
+    return f"every {signal.sampling_interval:g} s"
+
+
+def _signal_line(signal: Signal) -> str:
+    parts = [f"{len(signal)} points", signal.unit or "unit unknown"]
+    if len(signal):
+        parts.append(f"{signal.time[0]:g}-{signal.time[-1]:g} min")
+    parts.append(_interval(signal))
+    parts.append(str(signal.values.dtype))
+    return f"  signal {signal.channel!r}: " + ", ".join(p for p in parts if p)
+
+
+def _ms_line(ms: MSData) -> str:
+    parts = [f"{len(ms)} scans", f"{int(ms.peak_offset[-1])} peaks", ms.spectrum_type]
+    if len(ms):
+        parts.append(f"{ms.retention_time[0]:g}-{ms.retention_time[-1]:g} min")
+    if len(ms.mz):
+        parts.append(f"m/z {float(np.min(ms.mz)):g}-{float(np.max(ms.mz)):g}")
+    parts.append(f"m/z {ms.mz.dtype}, intensity {ms.intensity.dtype}")
+    if ms.flags is not None:
+        parts.append(f"{int(np.count_nonzero(ms.flags))} flagged peaks")
+    return f"  ms {ms.name!r}: " + ", ".join(parts)
+
+
+def describe(path: Path, fmt: str, injections: list[Injection]) -> str:
+    """Return the text printed by ``chromparser info``."""
+    lines = [f"{path} ({fmt}): {len(injections)} injection(s)"]
+    for number, injection in enumerate(injections, start=1):
+        head = f"injection {number}"
+        if injection.name:
+            head += f" {injection.name!r}"
+        if injection.inject_time is not None:
+            head += f", injected {injection.inject_time.isoformat()}"
+        lines.append(head)
+        lines.extend(_signal_line(s) for s in injection.signals.values())
+        lines.extend(_ms_line(ms) for ms in injection.ms.values())
+        for warning in injection.metadata.get("warnings", []):
+            lines.append(f"  warning: {warning}")
+    return "\n".join(lines)
+
+
+def _convert_one(args, source: Path, options: dict) -> list[Path]:
+    if args.to == "andi":
+        folder = args.output or source.parent
+        return write_andi(read(source, **options), folder, overwrite=args.overwrite)
+    output = convert(
+        source,
+        args.output,
+        overwrite=args.overwrite,
+        compression=not args.no_compression,
+        **options,
+    )
+    return [output]
+
+
+def _convert(args) -> int:
+    if args.to == "hdf5" and args.output and len(args.inputs) > 1:
+        print("chromparser: --output needs a single input", file=sys.stderr)
+        return 2
+    options = {"ms": not args.no_ms}
+    if args.signal_unit:
+        options["signal_unit"] = args.signal_unit
+    status = 0
+    for source in args.inputs:
+        try:
+            outputs = _convert_one(args, source, options)
+        except (OSError, ChromparserError, ValueError) as e:
+            print(f"chromparser: {source}: {e}", file=sys.stderr)
+            status = 1
+            continue
+        if not outputs:
+            print(f"chromparser: {source}: no signals or MS data", file=sys.stderr)
+        for output in outputs:
+            print(f"{source} -> {output}")
+    return status
+
+
+def _info(args) -> int:
+    status = 0
+    for source in args.inputs:
+        try:
+            fmt = sniff(source)
+            injections = read(source, ms=not args.no_ms)
+        except (OSError, ChromparserError, ValueError) as e:
+            print(f"chromparser: {source}: {e}", file=sys.stderr)
+            status = 1
+            continue
+        print(describe(source, fmt, injections))
+    return status
+
+
+def _extract(args) -> int:
+    status = 0
+    for source in args.inputs:
+        try:
+            written = extract_ms_raw(source, args.directory, overwrite=args.overwrite)
+        except (OSError, ChromparserError) as e:
+            print(f"chromparser: {source}: {e}", file=sys.stderr)
+            status = 1
+            continue
+        if not written:
+            print(f"chromparser: {source}: no MS data", file=sys.stderr)
+        for path in written:
+            print(f"{source} -> {path}")
+    return status
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Run the command line interface.
+
+    Parameters
+    ----------
+    argv : list of str, optional
+        Arguments; defaults to `sys.argv`.
+
+    Returns
+    -------
+    int
+        Exit status.
+    """
+    parser = argparse.ArgumentParser(
+        prog="chromparser",
+        description="Read chromatography raw files and convert them to HDF5.",
+    )
+    parser.add_argument(
+        "--version", action="version", version=f"%(prog)s {__version__}"
+    )
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    p = commands.add_parser(
+        "convert", help="convert files to HDF5 (layout v3) or ANDI netCDF"
+    )
+    p.add_argument("inputs", nargs="+", type=Path, help="raw files, .D folders or .h5")
+    p.add_argument(
+        "--to",
+        choices=("hdf5", "andi"),
+        default="hdf5",
+        help="hdf5 (default): one .h5 file; andi: one .cdf per signal and MS run",
+    )
+    p.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        help="output file (hdf5, single input) or folder (andi; default: next to the input)",
+    )
+    p.add_argument("--overwrite", action="store_true", help="replace existing output")
+    p.add_argument(
+        "--no-compression", action="store_true", help="do not compress (hdf5)"
+    )
+    p.add_argument("--no-ms", action="store_true", help="skip MS data of .cmbx files")
+    p.add_argument("--signal-unit", help="unit of Chromeleon raw signal files, e.g. pA")
+    p.set_defaults(run=_convert)
+
+    p = commands.add_parser("info", help="print injections, channels and MS runs")
+    p.add_argument("inputs", nargs="+", type=Path, help="raw files, .D folders or .h5")
+    p.add_argument("--no-ms", action="store_true", help="skip MS data of .cmbx files")
+    p.set_defaults(run=_info)
+
+    p = commands.add_parser(
+        "extract-ms", help="write the Thermo .raw files embedded in .cmbx archives"
+    )
+    p.add_argument("inputs", nargs="+", type=Path, help=".cmbx files")
+    p.add_argument(
+        "-d",
+        "--directory",
+        type=Path,
+        help="output folder (default: next to the input)",
+    )
+    p.add_argument("--overwrite", action="store_true", help="replace existing files")
+    p.set_defaults(run=_extract)
+
+    args = parser.parse_args(argv)
+    return args.run(args)

chromparser/errors.py

@@ -0,0 +1,49 @@
+"""Exceptions and warnings raised by the readers.
+
+All exceptions derive from `ValueError`, so code written for cmbx-reader, which
+raised `ValueError`, keeps working.
+"""
+
+from __future__ import annotations
+
+import warnings
+from typing import Any
+
+
+class ChromparserError(ValueError):
+    """Base class of the errors raised by chromparser."""
+
+
+class UnsupportedFormat(ChromparserError):
+    """The file is not chromatography data in any format chromparser reads."""
+
+
+class UnsupportedVariant(ChromparserError):
+    """The format is recognised, but this variant is not supported.
+
+    The message names what was found (version, chunk type, format code, ...).
+    """
+
+
+class CorruptFile(ChromparserError):
+    """The file is truncated or inconsistent and cannot be read completely."""
+
+
+class ChromparserWarning(UserWarning):
+    """A recoverable oddity in a file, such as a TIC mismatch."""
+
+
+def warn(metadata: dict[str, Any], message: str, stacklevel: int = 3) -> None:
+    """Issue a `ChromparserWarning` and record it in `metadata["warnings"]`.
+
+    Parameters
+    ----------
+    metadata : dict
+        Metadata of the object the warning is about.
+    message : str
+        The warning.
+    stacklevel : int
+        Passed to `warnings.warn`.
+    """
+    warnings.warn(message, ChromparserWarning, stacklevel=stacklevel)
+    metadata.setdefault("warnings", []).append(message)

chromparser/hdf5.py

@@ -0,0 +1,428 @@
+"""HDF5 files: write layout v3, read layouts 1-3.
+
+Layout v3 (the normative description is `docs/hdf5.md`)::
+
+    /                        attrs: format = "chromparser", layout_version = 3,
+                                    created_by, created, source, source_format,
+                                    source_sha256, source_size
+    /injection_0001          attrs: name, inject_time, <metadata>, metadata_json
+    /injection_0001/<signal> attrs: kind = "signal", channel, unit, acquisition_start,
+                                    sampling_interval, <metadata>, metadata_json
+        time                 float64, minutes (attr unit = "min")
+        signal               dtype as stored (attr unit, if known)
+    /injection_0001/<ms>     attrs: kind = "ms", name, spectrum_type,
+                                    acquisition_start, <metadata>, metadata_json
+        scan_number          int32
+        retention_time       float64, minutes
+        tic                  float64
+        peak_offset          int64, n_scans + 1
+        mz, intensity        as stored
+        base_peak_mz, base_peak_intensity, filter, ms_level, flags   optional
+
+Layout 2 is cmbx-reader's (the same without `metadata_json` and the optional
+MS fields), layout 1 that of cmbx-reader 0.1.0 (signals only, no `kind`).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from collections.abc import Iterable
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import h5py
+import numpy as np
+
+from ._util import head
+from .errors import UnsupportedFormat, UnsupportedVariant
+from .model import Injection, MSData, Signal
+
+FORMAT = "chromparser"
+LAYOUT_VERSION = 3
+READABLE_LAYOUTS = (1, 2, 3)
+HDF5_MAGIC = b"\x89HDF\r\n\x1a\n"
+
+PEAK_CHUNK = 1 << 20  # elements per chunk of the peak datasets
+
+_INJECTION_ATTRS = {"name", "inject_time", "metadata_json"}
+_SIGNAL_ATTRS = {
+    "kind",
+    "channel",
+    "unit",
+    "acquisition_start",
+    "sampling_interval",
+    "metadata_json",
+}
+_MS_ATTRS = {"kind", "name", "spectrum_type", "acquisition_start", "metadata_json"}
+
+
+# ------------------------------------------------------------------ writing
+def _is_scalar(value: Any) -> bool:
+    if isinstance(value, (bool, np.bool_, str, float, np.floating)):
+        return True
+    if isinstance(value, (int, np.integer)):
+        return -(2**63) <= int(value) < 2**63
+    return False
+
+
+def _json_default(value: Any) -> Any:
+    if isinstance(value, datetime):
+        return value.isoformat()
+    if isinstance(value, np.generic):
+        return value.item()
+    if isinstance(value, np.ndarray):
+        return value.tolist()
+    return str(value)
+
+
+def _set_attrs(obj: h5py.HLObject, attrs: dict, metadata: dict, reserved: set) -> None:
+    """Write fixed attributes, scalar metadata and `metadata_json`.
+
+    Metadata that is not scalar, or whose key is a reserved attribute name,
+    goes to `metadata_json`. `None` values are omitted.
+    """
+    for key, value in attrs.items():
+        if value is None:
+            continue
+        obj.attrs[key] = value.isoformat() if isinstance(value, datetime) else value
+    nested = {}
+    for key, value in metadata.items():
+        if value is None:
+            continue
+        if key in reserved or not _is_scalar(value):
+            nested[key] = value
+        else:
+            obj.attrs[key] = value
+    if nested:
+        obj.attrs["metadata_json"] = json.dumps(
+            nested, default=_json_default, ensure_ascii=False
+        )
+
+
+def _group_name(name: str, taken: set[str]) -> str:
+    """Sanitise a channel or MS name as an HDF5 group name, unique in `taken`."""
+    base = name.replace("/", "_")
+    if base in ("", ".", ".."):
+        base = "unnamed"
+    candidate, n = base, 1
+    while candidate in taken:
+        n += 1
+        candidate = f"{base}_{n}"
+    taken.add(candidate)
+    return candidate
+
+
+def _dataset(
+    group: h5py.Group, name: str, data: np.ndarray, compress: bool, chunk: int | None
+) -> h5py.Dataset:
+    kwargs: dict[str, Any] = {}
+    if compress and data.size:
+        kwargs = {"compression": "gzip", "compression_opts": 4}
+        if data.dtype.kind != "O":
+            kwargs["shuffle"] = True
+        kwargs["chunks"] = (min(data.size, chunk or PEAK_CHUNK),)
+    return group.create_dataset(name, data=data, **kwargs)
+
+
+def _write_signal(parent: h5py.Group, name: str, signal: Signal, compress: bool):
+    group = parent.create_group(name)
+    _set_attrs(
+        group,
+        {
+            "kind": "signal",
+            "channel": signal.channel,
+            "unit": signal.unit,
+            "acquisition_start": signal.acquisition_start,
+            "sampling_interval": signal.sampling_interval,
+        },
+        signal.metadata,
+        _SIGNAL_ATTRS,
+    )
+    time = _dataset(group, "time", np.asarray(signal.time), compress, None)
+    time.attrs["unit"] = "min"
+    values = _dataset(group, "signal", np.asarray(signal.values), compress, None)
+    if signal.unit:
+        values.attrs["unit"] = signal.unit
+
+
+def _write_ms(parent: h5py.Group, name: str, ms: MSData, compress: bool) -> None:
+    group = parent.create_group(name)
+    _set_attrs(
+        group,
+        {
+            "kind": "ms",
+            "name": ms.name,
+            "spectrum_type": ms.spectrum_type,
+            "acquisition_start": ms.acquisition_start,
+        },
+        ms.metadata,
+        _MS_ATTRS,
+    )
+    per_scan = {
+        "scan_number": ms.scan_number,
+        "retention_time": ms.retention_time,
+        "tic": ms.tic,
+        "peak_offset": ms.peak_offset,
+        "base_peak_mz": ms.base_peak_mz,
+        "base_peak_intensity": ms.base_peak_intensity,
+        "ms_level": ms.ms_level,
+    }
+    for key, data in per_scan.items():
+        if data is not None:
+            _dataset(group, key, np.asarray(data), compress, None)
+    if ms.filters is not None:
+        filters = np.array(ms.filters, dtype=h5py.string_dtype())
+        _dataset(group, "filter", filters, compress, None)
+    for key in ("mz", "intensity", "flags"):
+        data = getattr(ms, key)
+        if data is not None:
+            _dataset(group, key, np.asarray(data), compress, PEAK_CHUNK)
+    group["retention_time"].attrs["unit"] = "min"
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as f:
+        for block in iter(lambda: f.read(1 << 20), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def write_hdf5(
+    injections: Injection | Iterable[Injection],
+    path: str | Path,
+    *,
+    overwrite: bool = False,
+    compression: bool = True,
+    source: str | Path | None = None,
+    source_format: str | None = None,
+) -> Path:
+    """Write injections to an HDF5 file in layout v3.
+
+    Parameters
+    ----------
+    injections : Injection or iterable of Injection
+        What to write.
+    path : str or Path
+        Output file.
+    overwrite : bool
+        Replace an existing file.
+    compression : bool
+        Compress datasets (gzip level 4 with shuffle).
+    source : str or Path, optional
+        The input file. Its name is stored, and for a file also its size and
+        SHA-256.
+    source_format : str, optional
+        Format id of the input, e.g. ``"shimadzu-qgd"`` (see `sniff`).
+
+    Returns
+    -------
+    Path
+        The written file.
+
+    Raises
+    ------
+    FileExistsError
+        If the file exists and `overwrite` is False.
+    """
+    from . import __version__
+
+    if isinstance(injections, Injection):
+        injections = [injections]
+    path = Path(path)
+    if path.exists() and not overwrite:
+        raise FileExistsError(f"{path} exists, use overwrite=True to replace it.")
+
+    root = {
+        "format": FORMAT,
+        "layout_version": LAYOUT_VERSION,
+        "created_by": f"chromparser {__version__}",
+        "created": datetime.now(timezone.utc).replace(microsecond=0),
+        "source_format": source_format,
+    }
+    if source is not None:
+        source = Path(source)
+        root["source"] = source.name
+        if source.is_file():
+            root["source_sha256"] = _sha256(source)
+            root["source_size"] = source.stat().st_size
+
+    with h5py.File(path, "w") as hdf:
+        _set_attrs(hdf, root, {}, set())
+        for number, injection in enumerate(injections, start=1):
+            group = hdf.create_group(f"injection_{number:04d}")
+            _set_attrs(
+                group,
+                {"name": injection.name or None, "inject_time": injection.inject_time},
+                injection.metadata,
+                _INJECTION_ATTRS,
+            )
+            taken: set[str] = set()
+            for signal in injection.signals.values():
+                name = _group_name(signal.channel, taken)
+                _write_signal(group, name, signal, compression)
+            for ms in injection.ms.values():
+                _write_ms(group, _group_name(ms.name, taken), ms, compression)
+    return path
+
+
+# ------------------------------------------------------------------ reading
+def _value(value: Any) -> Any:
+    if isinstance(value, np.generic):
+        return value.item()
+    if isinstance(value, bytes):
+        return value.decode("utf-8")
+    if isinstance(value, np.ndarray):
+        return [_value(v) for v in value.tolist()]
+    return value
+
+
+def _attrs(obj: h5py.HLObject, reserved: set) -> tuple[dict, dict]:
+    """Split attributes into (reserved attributes, metadata)."""
+    fixed, metadata = {}, {}
+    for key, value in obj.attrs.items():
+        value = _value(value)
+        (fixed if key in reserved else metadata)[key] = value
+    if "metadata_json" in fixed:
+        metadata.update(json.loads(fixed.pop("metadata_json")))
+    return fixed, metadata
+
+
+def _time(text: str | None) -> datetime | None:
+    return datetime.fromisoformat(text) if text else None
+
+
+def _read_signal(group: h5py.Group) -> Signal:
+    fixed, metadata = _attrs(group, _SIGNAL_ATTRS)
+    return Signal(
+        channel=fixed.get("channel", group.name.rsplit("/", 1)[-1]),
+        time=group["time"][()],
+        values=group["signal"][()],
+        unit=fixed.get("unit"),
+        acquisition_start=_time(fixed.get("acquisition_start")),
+        metadata=metadata,
+        sampling_interval=fixed.get("sampling_interval"),
+    )
+
+
+def _read_ms(group: h5py.Group) -> MSData:
+    fixed, metadata = _attrs(group, _MS_ATTRS)
+
+    def optional(key):
+        return group[key][()] if key in group else None
+
+    return MSData(
+        name=fixed.get("name", group.name.rsplit("/", 1)[-1]),
+        scan_number=group["scan_number"][()],
+        retention_time=group["retention_time"][()],
+        tic=group["tic"][()],
+        base_peak_mz=optional("base_peak_mz"),
+        base_peak_intensity=optional("base_peak_intensity"),
+        filters=group["filter"].asstr()[()].tolist() if "filter" in group else None,
+        mz=group["mz"][()],
+        intensity=group["intensity"][()],
+        peak_offset=group["peak_offset"][()],
+        acquisition_start=_time(fixed.get("acquisition_start")),
+        metadata=metadata,
+        ms_level=optional("ms_level"),
+        flags=optional("flags"),
+        spectrum_type=fixed.get("spectrum_type", "centroid"),
+    )
+
+
+def layout_version(path: str | Path) -> int:
+    """Return the layout version of a chromparser or cmbx-reader HDF5 file.
+
+    Raises
+    ------
+    UnsupportedFormat
+        If the file is not such an HDF5 file.
+    """
+    if head(path, 8) != HDF5_MAGIC:
+        raise UnsupportedFormat(f"{path} is not an HDF5 file.")
+    with h5py.File(path, "r") as hdf:
+        version = _value(hdf.attrs.get("layout_version"))
+        created_by = str(_value(hdf.attrs.get("created_by", "")))
+        known = hdf.attrs.get("format") == FORMAT or created_by.startswith(
+            "cmbx-reader"
+        )
+    if not known or not isinstance(version, int):
+        raise UnsupportedFormat(
+            f"{path} is not a chromparser or cmbx-reader HDF5 file."
+        )
+    return version
+
+
+def read_hdf5(path: str | Path) -> list[Injection]:
+    """Read an HDF5 file written by chromparser (layout 3) or cmbx-reader (1, 2).
+
+    Parameters
+    ----------
+    path : str or Path
+        The HDF5 file.
+
+    Returns
+    -------
+    list of Injection
+        In file order. Groups are recognised by their `kind` attribute
+        (layout 1: by their `time` and `signal` datasets); unknown groups,
+        datasets and attributes are ignored.
+
+    Raises
+    ------
+    UnsupportedFormat
+        If the file is not a chromparser or cmbx-reader HDF5 file.
+    UnsupportedVariant
+        If its layout version is newer than this reader.
+    """
+    path = Path(path)
+    if not path.is_file():
+        raise FileNotFoundError(f"File not found: {path}")
+    version = layout_version(path)
+    if version not in READABLE_LAYOUTS:
+        raise UnsupportedVariant(
+            f"{path.name} has HDF5 layout version {version}; this chromparser "
+            f"reads {', '.join(map(str, READABLE_LAYOUTS))}. Update chromparser."
+        )
+    injections = []
+    with h5py.File(path, "r") as hdf:
+        names = sorted(
+            (n for n in hdf if re.fullmatch(r"injection_\d+", n)),
+            key=lambda n: int(n[10:]),
+        )
+        for name in names:
+            group = hdf[name]
+            fixed, metadata = _attrs(group, _INJECTION_ATTRS)
+            injection = Injection(
+                name=fixed.get("name", ""),
+                inject_time=_time(fixed.get("inject_time")),
+                metadata=metadata,
+            )
+            for child in group.values():
+                if not isinstance(child, h5py.Group):
+                    continue
+                kind = _value(child.attrs.get("kind"))
+                if kind is None and {"time", "signal"} <= child.keys():
+                    kind = "signal"
+                if kind == "signal":
+                    signal = _read_signal(child)
+                    injection.signals[signal.channel] = signal
+                elif kind == "ms":
+                    ms = _read_ms(child)
+                    injection.ms[ms.name] = ms
+            injections.append(injection)
+    return injections
+
+
+def sniff_hdf5(path: str | Path) -> bool:
+    """Whether `path` is an HDF5 file written by chromparser or cmbx-reader."""
+    if not Path(path).is_file():
+        return False
+    try:
+        layout_version(path)
+    except (UnsupportedFormat, OSError):
+        return False
+    return True