chromparser 0.1.0__py3-none-any.whl

chromparser/vendors/chromeleon/txt.py

@@ -0,0 +1,319 @@
+"""Chromeleon text exports of one chromatogram (`.txt`).
+
+The export is a tab-separated text file: ``key<TAB>value`` lines in sections
+(``Injection Information:``, ``Chromatogram Data Information:``, ...), then
+``Chromatogram Data:``, a column header and one ``time<TAB>step<TAB>value``
+row per point. Labels, number and date formats depend on the Chromeleon
+version and on the locale of the exporting computer; see
+`docs/formats/chromeleon.md`.
+"""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import numpy as np
+
+from ..._util import clean, existing_file, head, parse_iso_datetime
+from ...errors import CorruptFile, UnsupportedFormat, warn
+from ...model import Injection, Signal
+
+_DATA_MARKER = "Chromatogram Data:"
+
+# labels of different Chromeleon versions -> metadata keys
+_INJECTION_KEYS = {
+    "URL": "url",
+    "File Path": "url",
+    "Data Vault": "data_vault",
+    "Sequence": "sequence",
+    "Number": "number",
+    "Injection Number": "number",
+    "Position": "position",
+    "Comment": "comment",
+    "Processing Method": "processing_method",
+    "Instrument Method": "instrument_method",
+    "Type": "type",
+    "Status": "status",
+    "Operator": "operator",
+}
+_SIGNAL_KEYS = {
+    "Detector": "detector",
+    "Driver Name": "driver",
+    "Generating Data System": "generating_data_system",
+    "Exporting Data System": "exporting_data_system",
+    "Signal Quantity": "signal_quantity",
+    "Channel Type": "channel_type",
+    "Signal Info": "signal_info",
+}
+_NAME_KEYS = ("Name", "Injection")
+_TIME_KEYS = ("Inject Time", "Injection Time")
+# statistics of the data, checked or recomputable, not kept as metadata
+_DERIVED_KEYS = {
+    "Time Min. (min)",
+    "Time Max. (min)",
+    "Data Points",
+    "Signal Min.",
+    "Signal Max.",
+    "Min. Step (s)",
+    "Max. Step (s)",
+    "Average Step (s)",
+    "Channel",
+    "Signal Unit",
+    "Injection Date",
+}
+_SIGNAL_SECTIONS = ("Chromatogram Data Information", "Signal Parameter Information")
+
+
+def _snake(label: str) -> str:
+    label = label.replace("µ", "u").replace("μ", "u").lower()
+    return re.sub(r"[^a-z0-9]+", "_", label).strip("_")
+
+
+def _read_text(path: Path) -> list[str]:
+    data = path.read_bytes()
+    try:
+        text = data.decode("utf-8-sig")
+    except UnicodeDecodeError:
+        text = data.decode("cp1252")
+    return text.splitlines()
+
+
+def _header(lines: list[str]) -> tuple[list[tuple[str, str, str]], int | None]:
+    """Parse the ``key<TAB>value`` lines.
+
+    Returns
+    -------
+    entries : list of tuple
+        ``(section, key, value)`` with stripped strings.
+    data_start : int or None
+        Index of the first data row, `None` without ``Chromatogram Data:``.
+    """
+    entries = []
+    section = ""
+    for i, line in enumerate(lines):
+        stripped = line.strip()
+        if not stripped:
+            continue
+        if stripped.startswith(_DATA_MARKER):
+            return entries, i + 2  # skip the column header
+        if "\t" in stripped:
+            key, value = stripped.split("\t", 1)
+            entries.append((section, key.strip().rstrip(":"), value.strip()))
+        elif stripped.endswith(":"):
+            section = stripped[:-1]
+    return entries, None
+
+
+def _to_float(column: np.ndarray, comma_decimal: bool) -> np.ndarray:
+    """Convert number strings (``1,234.5`` or ``1.234,5``, ``n.a.``) to float64."""
+    column = np.char.strip(column)
+    column = np.where(np.isin(column, ["n.a.", ""]), "nan", column)
+    if comma_decimal:
+        column = np.char.replace(np.char.replace(column, ".", ""), ",", ".")
+    else:
+        column = np.char.replace(column, ",", "")
+    return column.astype(np.float64)
+
+
+def _number(text: str | None, comma_decimal: bool) -> float | None:
+    if not text:
+        return None
+    try:
+        return float(_to_float(np.array([text]), comma_decimal)[0])
+    except ValueError:
+        return None
+
+
+_DATE_TIME = re.compile(
+    r"^(\d{1,2})[/.-](\d{1,2})[/.-](\d{4})\s+(\d{1,2}):(\d{2}):(\d{2})"
+    r"(?:\s*([AaPp][Mm]))?(?:\s*([+-])(\d{2}):?(\d{2}))?$"
+)
+
+
+def _hour(hour: int, am_pm: str | None) -> int:
+    if not am_pm:
+        return hour
+    return hour % 12 + (12 if am_pm.upper() == "PM" else 0)
+
+
+def parse_inject_time(
+    text: str, date: str | None = None, dayfirst: bool = True
+) -> datetime:
+    """Parse the inject time of a Chromeleon text export.
+
+    The format follows the locale of the exporting computer. Day and month
+    order is read from the data when one of them is larger than 12; dates
+    with an AM/PM time are month first (US locale); otherwise `dayfirst`
+    decides. Exports of Chromeleon 7.2 store the date separately
+    (``Injection Date``).
+
+    Parameters
+    ----------
+    text : str
+        ``Inject Time`` (e.g. ``7/17/2023 3:35:22 PM +02:00``) or
+        ``Injection Time`` (e.g. ``1:43:35 PM`` or ``10:30:07``).
+    date : str, optional
+        ``Injection Date`` (``12/19/2023`` or ``22-Aug-25``), required when
+        `text` is only a time.
+    dayfirst : bool
+        Order for ambiguous dates such as ``7/1/2023``.
+
+    Returns
+    -------
+    datetime
+        Timezone-aware when the export has a UTC offset, naive otherwise.
+
+    Raises
+    ------
+    ValueError
+        If the time cannot be parsed.
+    """
+    text = text.strip()
+    match = _DATE_TIME.match(text)
+    if match:
+        first, second, year, hour, minute, sec, am_pm, sign, oh, om = match.groups()
+        first, second = int(first), int(second)
+        if first > 12:
+            day_first = True
+        elif second > 12 or am_pm:
+            # a 12-hour clock is the US locale, which writes the month first
+            day_first = False
+        else:
+            day_first = dayfirst
+        day, month = (first, second) if day_first else (second, first)
+        tz = None
+        if sign:
+            offset = timedelta(hours=int(oh), minutes=int(om))
+            tz = timezone(-offset if sign == "-" else offset)
+        return datetime(
+            int(year), month, day, _hour(int(hour), am_pm), int(minute), int(sec),
+            tzinfo=tz,
+        )  # fmt: skip
+
+    if re.match(r"\d{1,2}:\d{2}:\d{2} (AM|PM)", text):
+        if not date:
+            raise ValueError("Injection Date is missing for an AM/PM time.")
+        return datetime.strptime(f"{date} {text}", "%m/%d/%Y %I:%M:%S %p")
+    if re.match(r"\d{2}:\d{2}:\d{2}$", text):
+        if not date:
+            raise ValueError("Injection Date is missing for a time without date.")
+        if re.match(r"\d{1,2}-[A-Za-z]{3}-\d{2}$", date):
+            return datetime.strptime(f"{date} {text}", "%d-%b-%y %H:%M:%S")
+        if re.match(r"\d{1,2}/\d{1,2}/\d{4}$", date):
+            return datetime.strptime(f"{date} {text}", "%m/%d/%Y %H:%M:%S")
+        raise ValueError(f"Unrecognised Injection Date format: {date}")
+    parsed = parse_iso_datetime(text)
+    if parsed is None:
+        raise ValueError(f"Unrecognised Inject Time format: {text}")
+    return parsed
+
+
+def read_chromeleon_txt(path: str | Path, *, dayfirst: bool = True) -> list[Injection]:
+    """Read a Chromeleon text export of one chromatogram.
+
+    Parameters
+    ----------
+    path : str or Path
+        The `.txt` export.
+    dayfirst : bool
+        Day/month order for ambiguous dates (see `parse_inject_time`).
+
+    Returns
+    -------
+    list of Injection
+        One injection with one signal.
+
+    Raises
+    ------
+    UnsupportedFormat
+        If the file is not a Chromeleon chromatogram export.
+    CorruptFile
+        If the data rows cannot be read.
+    """
+    path = existing_file(path)
+    lines = _read_text(path)
+    entries, data_start = _header(lines)
+    if data_start is None:
+        raise UnsupportedFormat(f"{path.name}: no 'Chromatogram Data:' section.")
+    raw = {key: value for _, key, value in entries}
+
+    rows = [line.split("\t") for line in lines[data_start:] if line.strip()]
+    if any(len(row) < 3 for row in rows):
+        raise CorruptFile(f"{path.name}: data row with fewer than three columns.")
+    table = np.array([row[:3] for row in rows], dtype=str).reshape(-1, 3)
+    comma_decimal = bool(len(table)) and "," in table[0, 0] and "." not in table[0, 0]
+    try:
+        time = _to_float(table[:, 0], comma_decimal)
+        values = _to_float(table[:, 2], comma_decimal)
+    except ValueError as e:
+        raise CorruptFile(f"{path.name}: unreadable data row: {e}") from None
+
+    injection_metadata: dict = {}
+    signal_metadata: dict = {}
+    for section, key, value in entries:
+        if key in _DERIVED_KEYS or key in _NAME_KEYS or key in _TIME_KEYS:
+            continue
+        if key in _INJECTION_KEYS:
+            injection_metadata[_INJECTION_KEYS[key]] = value
+        elif key in _SIGNAL_KEYS or section in _SIGNAL_SECTIONS:
+            signal_metadata[_SIGNAL_KEYS.get(key, _snake(key))] = value
+        else:
+            injection_metadata[_snake(key)] = value
+    url = injection_metadata.get("url", "")
+    if url.startswith("chrom://"):
+        signal_metadata["url"] = url
+
+    inject_time = None
+    time_key = next((key for key in _TIME_KEYS if raw.get(key)), None)
+    if time_key is None:
+        warn(injection_metadata, f"{path.name}: no Inject Time.")
+    else:
+        try:
+            inject_time = parse_inject_time(
+                raw[time_key], raw.get("Injection Date"), dayfirst
+            )
+        except ValueError as e:
+            injection_metadata["chromeleon_inject_time"] = raw[time_key]
+            warn(injection_metadata, f"{path.name}: {e}")
+
+    expected = _number(raw.get("Data Points"), False)
+    if expected is not None and expected != len(time):
+        warn(
+            signal_metadata,
+            f"{path.name}: {len(time)} data rows, the header lists {int(expected)}.",
+        )
+    steps = [
+        _number(raw.get(key), comma_decimal)
+        for key in ("Min. Step (s)", "Max. Step (s)", "Average Step (s)")
+    ]
+    interval = steps[2] if steps[0] is not None and len(set(steps)) == 1 else None
+
+    signal = Signal(
+        channel=raw.get("Channel", ""),
+        time=time,
+        values=values,
+        unit=raw.get("Signal Unit") or None,
+        metadata=clean(signal_metadata),
+        sampling_interval=interval,
+    )
+    name = next((raw[key] for key in _NAME_KEYS if key in raw), "")
+    return [
+        Injection(
+            name=name,
+            inject_time=inject_time,
+            signals={signal.channel: signal},
+            metadata=clean(injection_metadata),
+        )
+    ]
+
+
+def sniff_chromeleon_txt(path: str | Path) -> bool:
+    """Whether `path` looks like a Chromeleon chromatogram text export."""
+    if not Path(path).is_file():
+        return False
+    text = head(path, 1 << 16).decode("utf-8", errors="replace")
+    return (
+        "\nChromatogram Data:" in text or "\nChromatogram Data Information:" in text
+    ) and "Injection Information:" in text

chromparser/vendors/shimadzu/__init__.py

@@ -0,0 +1,8 @@
+"""Shimadzu GCMSsolution: `.qgd` (GC-MS) and `.gcd` (GC) files."""
+
+from __future__ import annotations
+
+from .gcd import decode_gcd, read_gcd, sniff_gcd
+from .qgd import decode_qgd, read_qgd, sniff_qgd
+
+__all__ = ["decode_gcd", "decode_qgd", "read_gcd", "read_qgd", "sniff_gcd", "sniff_qgd"]

chromparser/vendors/shimadzu/gcd.py

@@ -0,0 +1,146 @@
+"""Shimadzu GCMSsolution GC files (`.gcd`, storages ``GC Raw Data-<n>``).
+
+Each storage holds one detector channel: ``Intensity Data`` (float64 per
+sample), ``Intensity Data Flag`` (one byte per sample) and ``Status``
+(sampling interval, start, number of samples). LabSolutions GC files store
+``LSS Raw Data`` instead; they are recognised and raise `UnsupportedVariant`.
+See `docs/formats/shimadzu.md`.
+"""
+
+from __future__ import annotations
+
+import struct
+from pathlib import Path
+
+import numpy as np
+
+from ...errors import CorruptFile, UnsupportedFormat, UnsupportedVariant, warn
+from ...model import Injection, Signal
+from .ole import (
+    GCD_PREFIX,
+    LSS_STORAGE,
+    file_properties,
+    open_ole,
+    read_stream,
+    storages,
+)
+
+FLAG_VALUE = 0x80  # the only flag value seen; its meaning is unknown
+
+
+def decode_gcd(
+    streams: dict[str, bytes], channel: str = "1", name: str = "gcd"
+) -> Signal:
+    """Decode the streams of one ``GC Raw Data-<n>`` storage.
+
+    Parameters
+    ----------
+    streams : dict of str to bytes
+        ``"Intensity Data"``, ``"Status"`` and optionally
+        ``"Intensity Data Flag"``.
+    channel : str
+        Channel name.
+    name : str
+        File name for messages.
+
+    Returns
+    -------
+    Signal
+        Values as stored (float64), unit unknown.
+
+    Raises
+    ------
+    CorruptFile
+        If the status is invalid.
+    """
+    status = streams["Status"]
+    if len(status) < 12:
+        raise CorruptFile(f"{name}: Status of channel {channel} is truncated.")
+    interval_ms, start_ms, n = struct.unpack_from("<iii", status)
+    if interval_ms <= 0:
+        raise CorruptFile(f"{name}: invalid sampling interval {interval_ms} ms.")
+    data = streams["Intensity Data"]
+    if len(data) % 8:
+        raise CorruptFile(f"{name}: Intensity Data is not a float64 array.")
+    values = np.frombuffer(data, "<f8").astype(np.float64)
+    metadata: dict = {"shimadzu_storage": f"{GCD_PREFIX}{channel}"}
+    if n != len(values):
+        warn(metadata, f"{name}: {len(values)} samples stored, {n} expected.")
+    flags = streams.get("Intensity Data Flag")
+    if flags is not None and np.any(np.frombuffer(flags, np.uint8) != FLAG_VALUE):
+        warn(metadata, f"{name}: intensity flags of unknown meaning are not read.")
+    if start_ms:
+        metadata["shimadzu_start_ms"] = start_ms
+    time = (start_ms + interval_ms * np.arange(len(values), dtype=np.float64)) / 60000.0
+    return Signal(
+        channel=channel,
+        time=time,
+        values=values,
+        metadata=metadata,
+        sampling_interval=interval_ms / 1000.0,
+    )
+
+
+def read_gcd(path: str | Path) -> list[Injection]:
+    """Read the chromatograms of a Shimadzu GCMSsolution `.gcd` file.
+
+    Parameters
+    ----------
+    path : str or Path
+        The `.gcd` file.
+
+    Returns
+    -------
+    list of Injection
+        One injection with one signal per ``GC Raw Data-<n>`` storage, named
+        ``<n>``; empty storages (channels without data) are skipped. `inject_time` is `None`; the acquisition start (UTC) is in
+        `Signal.acquisition_start`.
+
+    Raises
+    ------
+    UnsupportedVariant
+        For LabSolutions files (``LSS Raw Data``).
+    UnsupportedFormat
+        If the file holds no GC raw data.
+    """
+    path = Path(path)
+    with open_ole(path) as ole:
+        names = sorted(
+            {e[0] for e in ole.listdir(storages=True) if e[0].startswith(GCD_PREFIX)}
+        )
+        if not names:
+            if ole.exists(LSS_STORAGE):
+                raise UnsupportedVariant(
+                    f"{path.name} is a LabSolutions GC file ({LSS_STORAGE}), which "
+                    "is not supported yet. Please share a sample file."
+                )
+            raise UnsupportedFormat(f"{path.name} contains no GC raw data.")
+        start, metadata = file_properties(ole)
+        signals = {}
+        for storage in names:
+            streams = {
+                stream: read_stream(ole, f"{storage}/{stream}")
+                for stream in ("Intensity Data", "Status", "Intensity Data Flag")
+                if ole.exists(f"{storage}/{stream}")
+            }
+            channel = storage[len(GCD_PREFIX) :]
+            if not streams:
+                continue  # an empty storage: a configured channel without data
+            if {"Intensity Data", "Status"} - streams.keys():
+                raise CorruptFile(f"{path.name}: {storage} is incomplete.")
+            signal = decode_gcd(streams, channel, path.name)
+            signal.acquisition_start = start
+            signals[channel] = signal
+    if not signals:
+        raise UnsupportedFormat(f"{path.name}: all GC raw data storages are empty.")
+    return [
+        Injection(
+            name=metadata.get("sample_name", ""), signals=signals, metadata=metadata
+        )
+    ]
+
+
+def sniff_gcd(path: str | Path) -> bool:
+    """Whether `path` is an OLE2 file with GC raw data (GCMSsolution or LabSolutions)."""
+    names = storages(path)
+    return LSS_STORAGE in names or any(n.startswith(GCD_PREFIX) for n in names)

chromparser/vendors/shimadzu/ole.py

@@ -0,0 +1,84 @@
+"""OLE2 compound documents written by Shimadzu GCMSsolution.
+
+Both `.qgd` (GC-MS) and `.gcd` (GC) files are OLE2 compound documents
+("structured storage"), read with `olefile`. The ``File Property`` stream
+holds the acquisition start and the sample name.
+"""
+
+from __future__ import annotations
+
+import struct
+from contextlib import contextmanager
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import olefile
+
+from ..._util import existing_file, filetime, head
+from ...errors import UnsupportedFormat
+
+OLE_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
+QGD_STORAGE = "GCMS Raw Data"
+GCD_PREFIX = "GC Raw Data-"
+LSS_STORAGE = "LSS Raw Data"
+
+
+def is_ole_file(path: str | Path) -> bool:
+    """Whether `path` is an OLE2 compound document."""
+    return Path(path).is_file() and head(path, 8) == OLE_MAGIC
+
+
+@contextmanager
+def open_ole(path: str | Path):
+    """Open an OLE2 file, raising `UnsupportedFormat` if it is none."""
+    path = existing_file(path)
+    try:
+        ole = olefile.OleFileIO(str(path))
+    except OSError as e:  # olefile raises OSError for non-OLE files
+        raise UnsupportedFormat(f"{path.name} is not an OLE2 file: {e}") from e
+    with ole:
+        yield ole
+
+
+def storages(path: str | Path) -> set[str]:
+    """Names of the top-level storages and streams; empty for non-OLE files."""
+    if not is_ole_file(path):
+        return set()
+    try:
+        with open_ole(path) as ole:
+            return {entry[0] for entry in ole.listdir(streams=True, storages=True)}
+    except (UnsupportedFormat, OSError):
+        return set()
+
+
+def read_stream(ole: olefile.OleFileIO, name: str) -> bytes:
+    """Read a stream, given as ``storage/stream``."""
+    return ole.openstream(name).read()
+
+
+def file_properties(ole: olefile.OleFileIO) -> tuple[datetime | None, dict[str, Any]]:
+    """Read the ``File Property`` stream.
+
+    Returns
+    -------
+    acquisition_start : datetime or None
+        Windows FILETIME (UTC) at byte 52, equal to the creation time of the
+        raw data storage.
+    metadata : dict
+        ``sample_name`` (null-terminated, cp1252, at byte 204), if present.
+    """
+    if not ole.exists("File Property"):
+        return None, {}
+    data = read_stream(ole, "File Property")
+    start = None
+    metadata = {}
+    if len(data) >= 60:
+        (ticks,) = struct.unpack_from("<Q", data, 52)
+        if ticks:
+            start = filetime(ticks)
+    if len(data) > 204:
+        sample = data[204:300].split(b"\0", 1)[0].decode("cp1252", "replace").strip()
+        if sample:
+            metadata["sample_name"] = sample
+    return start, metadata