chromparser 0.1.0__py3-none-any.whl

chromparser/vendors/agilent/ch.py

@@ -0,0 +1,344 @@
+"""Agilent ChemStation / OpenLab chromatogram files (`.ch`).
+
+Supported file versions: 8 (and 80), 81, 179 and 181. The decoders are
+adapted from ``ImportAgilent.m`` in chemplexity/chromatography
+(https://github.com/chemplexity/chromatography, MIT License, Copyright (c)
+2014 James Dillon; see LICENSE), by way of chromstream's ``parsers/agilent.py``.
+See `docs/formats/agilent.md` for the layout.
+"""
+
+from __future__ import annotations
+
+import re
+import struct
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from ..._util import clean, existing_file, head
+from ...errors import CorruptFile, UnsupportedFormat, UnsupportedVariant, warn
+from ...model import Injection, Signal
+
+SUPPORTED_VERSIONS = ("8", "80", "81", "179", "181")
+
+# version -> byte offset of each Pascal string
+_STRINGS_V8 = {
+    "sample_name": 24,
+    "sample_description": 86,
+    "operator": 148,
+    "date": 178,
+    "inlet": 208,
+    "instrument": 218,
+    "method": 228,
+    "unit": 580,
+}
+_STRINGS_V179 = {
+    "sample_name": 858,
+    "sample_description": 1369,
+    "operator": 1880,
+    "date": 2391,
+    "inlet": 2492,
+    "instrument": 2533,
+    "method": 2574,
+    "unit": 4172,
+}
+_DATE_FORMATS = (
+    "%d %b %y %I:%M %p",
+    "%m/%d/%y %I:%M:%S %p",
+    "%d-%b-%y, %H:%M:%S",
+    "%d %b %y %H:%M:%S",
+)
+# version 8 files with header code 1-3 at byte 542 are scaled by this factor
+# instead of slope and intercept (as in ImportAgilent.m)
+_CODE_FACTOR = 1.33321110047553
+
+
+class _Buffer:
+    """Bounds-checked big-endian reads."""
+
+    def __init__(self, data: bytes, name: str):
+        self.data = data
+        self.name = name
+
+    def unpack(self, fmt: str, offset: int) -> tuple:
+        size = struct.calcsize(">" + fmt)
+        if offset + size > len(self.data):
+            raise CorruptFile(f"{self.name}: truncated header at offset {offset}.")
+        return struct.unpack_from(">" + fmt, self.data, offset)
+
+    def pascal(self, offset: int, utf16: bool) -> str:
+        """Read a Pascal string: length byte, then latin-1 or UTF-16 characters."""
+        if offset >= len(self.data):
+            raise CorruptFile(f"{self.name}: truncated header at offset {offset}.")
+        length = self.data[offset]
+        raw = self.data[offset + 1 : offset + 1 + length * (2 if utf16 else 1)]
+        try:
+            return raw.decode("utf-16-le" if utf16 else "latin-1").strip()
+        except UnicodeDecodeError:
+            return raw.decode("latin-1", errors="replace").strip()
+
+
+def decode_delta(data: bytes, offset: int) -> np.ndarray:
+    """Decode delta-compressed values (version 8).
+
+    Blocks of a big-endian int16 header (count in the low 12 bits, 0 in the
+    high 4 bits ends the data), then `count` int16 deltas. A delta of -32768
+    is followed by an absolute int32 value.
+
+    Parameters
+    ----------
+    data : bytes
+        The file.
+    offset : int
+        Start of the data.
+
+    Returns
+    -------
+    numpy.ndarray
+        The integer values as float64.
+    """
+    values = []
+    value = 0
+    pos, end = offset, len(data)
+    while pos + 2 <= end:
+        (header,) = struct.unpack_from(">h", data, pos)
+        pos += 2
+        if header >> 12 == 0:
+            break
+        for _ in range(header & 0x0FFF):
+            if pos + 2 > end:
+                raise CorruptFile("Agilent .ch: delta-compressed data is truncated.")
+            (delta,) = struct.unpack_from(">h", data, pos)
+            pos += 2
+            if delta != -32768:
+                value += delta
+            else:
+                if pos + 4 > end:
+                    raise CorruptFile(
+                        "Agilent .ch: delta-compressed data is truncated."
+                    )
+                (value,) = struct.unpack_from(">i", data, pos)
+                pos += 4
+            values.append(value)
+    return np.array(values, dtype=np.float64)
+
+
+def decode_double_delta(data: bytes, offset: int) -> np.ndarray:
+    """Decode double-delta-compressed values (versions 81 and 181).
+
+    Big-endian int16 second differences; 32767 is followed by an absolute
+    48-bit value (int16 high part, uint32 low part), which resets the delta.
+
+    Parameters
+    ----------
+    data : bytes
+        The file.
+    offset : int
+        Start of the data.
+
+    Returns
+    -------
+    numpy.ndarray
+        The integer values as float64.
+    """
+    values = []
+    signal = delta = 0
+    pos, end = offset, len(data)
+    while pos + 2 <= end:
+        (step,) = struct.unpack_from(">h", data, pos)
+        pos += 2
+        if step != 32767:
+            delta += step
+            signal += delta
+        else:
+            if pos + 6 > end:
+                raise CorruptFile("Agilent .ch: double-delta data is truncated.")
+            high, low = struct.unpack_from(">hI", data, pos)
+            pos += 6
+            signal = high * 4294967296 + low
+            delta = 0
+        values.append(signal)
+    return np.array(values, dtype=np.float64)
+
+
+def decode_double_array(data: bytes, offset: int) -> np.ndarray:
+    """Decode uncompressed little-endian float64 values (version 179)."""
+    count = max(len(data) - offset, 0) // 8
+    return np.frombuffer(data, dtype="<f8", count=count, offset=offset).astype(
+        np.float64
+    )
+
+
+def parse_date(text: str) -> datetime | None:
+    """Parse the date string of a `.ch` header (local time, no time zone).
+
+    Parameters
+    ----------
+    text : str
+        E.g. ``14-Jan-26, 11:49:50``.
+
+    Returns
+    -------
+    datetime or None
+        Naive datetime, `None` if the format is unknown.
+    """
+    for fmt in _DATE_FORMATS:
+        try:
+            return datetime.strptime(text, fmt)
+        except ValueError:
+            continue
+    return None
+
+
+def version_of(data: bytes) -> str | None:
+    """Return the version string at the start of a `.ch` file, if any."""
+    if len(data) < 4 or not 1 <= data[0] <= 3:
+        return None
+    version = data[1 : 1 + data[0]]
+    return version.decode("ascii") if re.fullmatch(rb"\d+", version) else None
+
+
+def decode_ch(
+    data: bytes, name: str = ".ch"
+) -> tuple[Signal, datetime | None, dict[str, Any]]:
+    """Decode an Agilent `.ch` file.
+
+    Parameters
+    ----------
+    data : bytes
+        Content of the file.
+    name : str
+        File name, used as channel name (its stem) and in messages.
+
+    Returns
+    -------
+    signal : Signal
+        The chromatogram, with the unit from the header.
+    inject_time : datetime or None
+        Naive local time from the header.
+    metadata : dict
+        Injection metadata (``sample_name``, ``operator``, ...).
+
+    Raises
+    ------
+    UnsupportedFormat
+        If the data is not an Agilent `.ch` file.
+    UnsupportedVariant
+        If the file version is not supported.
+    CorruptFile
+        If the file is truncated.
+    """
+    version = version_of(data)
+    if version is None:
+        raise UnsupportedFormat(f"{name} is not an Agilent .ch file.")
+    if version not in SUPPORTED_VERSIONS:
+        raise UnsupportedVariant(
+            f"{name}: Agilent .ch file version {version} is not supported "
+            f"(supported: {', '.join(SUPPORTED_VERSIONS)})."
+        )
+    buffer = _Buffer(data, name)
+    utf16 = version in ("179", "181")
+    offsets = _STRINGS_V179 if utf16 else _STRINGS_V8
+    strings = {key: buffer.pascal(offset, utf16) for key, offset in offsets.items()}
+    (block,) = buffer.unpack("i", 264)
+    data_offset = (block - 1) * 512
+    if not 0 < data_offset <= len(data):
+        raise CorruptFile(f"{name}: data offset {data_offset} is outside the file.")
+
+    if version in ("8", "80"):
+        values = decode_delta(data, data_offset)
+        start, end = buffer.unpack("ii", 282)
+        (code,) = buffer.unpack("i", 542)
+        if code in (1, 2, 3):
+            values = values * _CODE_FACTOR
+        else:
+            intercept, slope = buffer.unpack("dd", 636)
+            values = values * slope + intercept
+    else:
+        start, end = buffer.unpack("ff", 282)
+        intercept, slope = buffer.unpack("dd", 4724 if utf16 else 636)
+        if version == "179":
+            values = decode_double_array(data, data_offset)
+        else:
+            values = decode_double_delta(data, data_offset)
+        values = values * slope + intercept
+
+    # start and end of the run in ms
+    if len(values) > 1:
+        time = np.linspace(start / 60000.0, end / 60000.0, len(values))
+        interval = round((end - start) / (len(values) - 1) / 1000.0, 12)
+    else:
+        time = np.zeros(len(values))
+        interval = None
+
+    metadata = clean(
+        {
+            "sample_name": strings["sample_name"],
+            "agilent_sample_description": strings["sample_description"],
+            "operator": strings["operator"],
+            "instrument": strings["instrument"],
+            "agilent_inlet": strings["inlet"],
+            "instrument_method": strings["method"],
+        }
+    )
+    signal_metadata: dict[str, Any] = {"agilent_file_version": version}
+    if version in ("81", "181") and (len(data) - data_offset) % 2:
+        warn(signal_metadata, f"{name}: odd trailing byte after the data, ignored.")
+    inject_time = parse_date(strings["date"]) if strings["date"] else None
+    if strings["date"]:
+        metadata["agilent_date"] = strings["date"]
+        if inject_time is None:
+            warn(metadata, f"{name}: unknown date format {strings['date']!r}.")
+    signal = Signal(
+        channel=Path(name).stem,
+        time=time,
+        values=values,
+        unit=strings["unit"] or None,
+        metadata=signal_metadata,
+        sampling_interval=interval,
+    )
+    return signal, inject_time, metadata
+
+
+def read_agilent_ch(path: str | Path, *, channel: str | None = None) -> list[Injection]:
+    """Read an Agilent `.ch` file.
+
+    Parameters
+    ----------
+    path : str or Path
+        The `.ch` file.
+    channel : str, optional
+        Channel name; defaults to the file name without extension.
+
+    Returns
+    -------
+    list of Injection
+        One injection with one signal. `inject_time` is naive local time:
+        the file does not store the time zone.
+    """
+    path = existing_file(path)
+    signal, inject_time, metadata = decode_ch(path.read_bytes(), path.name)
+    if channel:
+        signal.channel = channel
+    return [
+        Injection(
+            name=metadata.get("sample_name", ""),
+            inject_time=inject_time,
+            signals={signal.channel: signal},
+            metadata=metadata,
+        )
+    ]
+
+
+def sniff_agilent_ch(path: str | Path) -> bool:
+    """Whether `path` starts like an Agilent `.ch` file (any version)."""
+    path = Path(path)
+    if not path.is_file() or path.stat().st_size < 512:
+        return False
+    data = head(path, 268)
+    if version_of(data) is None:
+        return False
+    (block,) = struct.unpack_from(">i", data, 264)
+    return 0 < (block - 1) * 512 <= path.stat().st_size

chromparser/vendors/agilent/dot_d.py

@@ -0,0 +1,69 @@
+"""Agilent data folders (`.D`): one injection, one `.ch` file per signal."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ...errors import UnsupportedFormat, warn
+from ...model import Injection
+from .ch import decode_ch, sniff_agilent_ch
+
+# files of an Agilent folder that hold data this reader does not decode
+_UNREAD = {".ms": "MS data", ".uv": "UV spectra"}
+
+
+def _ch_files(path: Path) -> list[Path]:
+    return sorted(
+        (p for p in path.iterdir() if p.is_file() and p.suffix.lower() == ".ch"),
+        key=lambda p: p.name.lower(),
+    )
+
+
+def read_agilent_d(path: str | Path) -> list[Injection]:
+    """Read the chromatograms (`.ch` files) of an Agilent `.D` folder.
+
+    Parameters
+    ----------
+    path : str or Path
+        The `.D` folder.
+
+    Returns
+    -------
+    list of Injection
+        One injection with one signal per `.ch` file, in file name order.
+        Injection metadata comes from the first file.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the folder does not exist.
+    UnsupportedFormat
+        If it holds no `.ch` files.
+    """
+    path = Path(path)
+    if not path.is_dir():
+        raise FileNotFoundError(f"Folder not found: {path}")
+    files = _ch_files(path)
+    if not files:
+        raise UnsupportedFormat(f"{path} holds no Agilent .ch files.")
+    injection = None
+    for file in files:
+        signal, inject_time, metadata = decode_ch(file.read_bytes(), file.name)
+        if injection is None:
+            injection = Injection(
+                name=metadata.get("sample_name", ""),
+                inject_time=inject_time,
+                metadata=metadata,
+            )
+        injection.signals[signal.channel] = signal
+    for p in sorted(path.iterdir()):
+        what = _UNREAD.get(p.suffix.lower())
+        if what:
+            warn(injection.metadata, f"{p.name}: {what} is not read (not supported).")
+    return [injection]
+
+
+def sniff_agilent_d(path: str | Path) -> bool:
+    """Whether `path` is a folder with at least one Agilent `.ch` file."""
+    path = Path(path)
+    return path.is_dir() and any(sniff_agilent_ch(p) for p in _ch_files(path))

chromparser/vendors/agilent/dx.py

@@ -0,0 +1,147 @@
+"""Agilent OpenLab CDS `.dx` files: a zip archive with `.ch` files and `.acmd`.
+
+The `injection.acmd` XML names the channel of each `.ch` member (by trace id)
+and holds the injection information, including the run time with its UTC
+offset. Instrument traces (`.IT` members: pressures, temperatures) are not
+read.
+"""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+import zipfile
+from pathlib import Path, PurePosixPath
+
+from ..._util import clean, existing_file, parse_iso_datetime
+from ...errors import UnsupportedFormat, warn
+from ...model import Injection
+from .ch import decode_ch, version_of
+
+_NS = {"acmd": "urn:schemas-agilent-com:acmd20"}
+
+
+def _acmd(archive: zipfile.ZipFile) -> tuple[dict, dict[str, dict]]:
+    """Parse the `.acmd` member.
+
+    Returns
+    -------
+    injection : dict
+        Texts of the ``InjectionInfo`` elements.
+    signals : dict of str to dict
+        Texts of each ``Signal`` element, by ``TraceId``.
+    """
+    name = next((n for n in archive.namelist() if n.lower().endswith(".acmd")), None)
+    if name is None:
+        return {}, {}
+    root = ET.fromstring(archive.read(name))
+    info = root.find("acmd:InjectionInfo", _NS)
+    injection: dict = {}
+    signals: dict[str, dict] = {}
+    if info is None:
+        return injection, signals
+    for child in info:
+        tag = child.tag.split("}")[-1]
+        if tag != "Signals" and child.text and child.text.strip():
+            injection[tag] = child.text.strip()
+    for signal in info.iterfind("acmd:Signals/acmd:Signal", _NS):
+        fields = {
+            child.tag.split("}")[-1]: (child.text or "").strip() for child in signal
+        }
+        if fields.get("TraceId"):
+            signals[fields["TraceId"].lower()] = fields
+    return injection, signals
+
+
+def read_agilent_dx(path: str | Path) -> list[Injection]:
+    """Read the chromatograms of an Agilent OpenLab `.dx` file.
+
+    Parameters
+    ----------
+    path : str or Path
+        The `.dx` file.
+
+    Returns
+    -------
+    list of Injection
+        One injection with one signal per `.ch` member, named after the
+        channel in the `.acmd` file. `inject_time` is timezone-aware.
+
+    Raises
+    ------
+    UnsupportedFormat
+        If the file is not a `.dx` archive or holds no `.ch` members.
+    """
+    path = existing_file(path)
+    try:
+        archive = zipfile.ZipFile(path)
+    except zipfile.BadZipFile as e:
+        raise UnsupportedFormat(f"'{path}' is not a .dx archive: {e}") from e
+    with archive:
+        info, trace_info = _acmd(archive)
+        members = sorted(
+            (n for n in archive.namelist() if n.lower().endswith(".ch")),
+            key=str.lower,
+        )
+        if not members:
+            raise UnsupportedFormat(f"{path.name} holds no .ch chromatograms.")
+        injection = None
+        for member in members:
+            signal, inject_time, metadata = decode_ch(archive.read(member), member)
+            trace = trace_info.get(PurePosixPath(member).stem.lower(), {})
+            signal.channel = trace.get("ChannelName") or signal.channel
+            signal.metadata.update(
+                clean(
+                    {
+                        "agilent_trace_id": trace.get("TraceId"),
+                        "detector": trace.get("DeviceName"),
+                        "agilent_description": trace.get("Description"),
+                    }
+                )
+            )
+            if injection is None:
+                run_time = parse_iso_datetime(info.get("RunDateTime", ""))
+                metadata.update(
+                    clean(
+                        {
+                            "sample_name": info.get("SampleName"),
+                            "operator": info.get("RunOperator"),
+                            "instrument_method": info.get("AcquisitionMethod"),
+                            "agilent_location": info.get("Location"),
+                            "agilent_injection_volume": info.get("InjectionVolume"),
+                            "agilent_injection_volume_unit": info.get(
+                                "InjectionVolumeUnits"
+                            ),
+                            "agilent_sequence_line": info.get("SequenceLine"),
+                            "agilent_run_date_time": info.get("RunDateTime"),
+                        }
+                    )
+                )
+                injection = Injection(
+                    name=metadata.get("sample_name", ""),
+                    inject_time=run_time or inject_time,
+                    metadata=metadata,
+                )
+            if signal.channel in injection.signals:
+                warn(
+                    injection.metadata,
+                    f"{path.name}: two signals named {signal.channel!r}; "
+                    f"{member} is stored under its member name.",
+                )
+                signal.channel = PurePosixPath(member).stem
+            injection.signals[signal.channel] = signal
+    return [injection]
+
+
+def sniff_agilent_dx(path: str | Path) -> bool:
+    """Whether `path` is a zip archive with an `.acmd` member or `.ch` members."""
+    if not Path(path).is_file() or not zipfile.is_zipfile(path):
+        return False
+    try:
+        with zipfile.ZipFile(path) as archive:
+            names = archive.namelist()
+            if any(n.lower().endswith(".acmd") for n in names):
+                return True
+            members = [n for n in names if n.lower().endswith(".ch")]
+            return bool(members) and version_of(archive.read(members[0])) is not None
+    except (zipfile.BadZipFile, OSError):
+        return False

chromparser/vendors/andi/__init__.py

@@ -0,0 +1,15 @@
+"""ANDI netCDF files: ANDI-MS (GC-MS) and ANDI chromatography (AIA)."""
+
+from __future__ import annotations
+
+from .chrom import read_andi_chrom, sniff_andi_chrom
+from .ms import read_andi_ms, sniff_andi_ms
+from .write import write_andi
+
+__all__ = [
+    "read_andi_chrom",
+    "read_andi_ms",
+    "sniff_andi_chrom",
+    "sniff_andi_ms",
+    "write_andi",
+]