chromparser 0.1.0__py3-none-any.whl

chromparser/vendors/chromeleon/raw.py

@@ -0,0 +1,407 @@
+"""Chromeleon 7 raw signal files (`SignHdr` / `PtsLL2Df` / `PtsDDCmp` chunks).
+
+These are the `.raw` files starting with `SignHdr`, found inside `.cmbx`
+archives and in the `RawData` folder of a data vault. See
+`docs/formats/chromeleon.md` for the binary layout.
+"""
+
+from __future__ import annotations
+
+import lzma
+import struct
+import uuid
+import warnings
+from pathlib import Path
+
+import numpy as np
+
+from ..._util import Source, existing_file, filetime, head, read_bytes
+from ...errors import (
+    ChromparserWarning,
+    CorruptFile,
+    UnsupportedFormat,
+    UnsupportedVariant,
+)
+from ...model import Injection, Signal
+
+SIGNAL_MAGIC = b"SignHdr\x00"
+POINTS_TAG = b"PtsLL2Df"
+COMPRESSED_POINTS_TAG = b"PtsDDCmp"
+AUDIT_TRAIL_MAGIC = b"CpAu"
+THERMO_RAW_MAGIC = b"\x01\xa1" + "Finnigan".encode("utf-16-le")
+
+_CHUNK_HEADER = struct.Struct("<8sQ")
+
+
+def read_svarint(buf: bytes, pos: int) -> tuple[int, int]:
+    """Read one signed varint.
+
+    The first byte is ``[continue][sign][6 value bits]``, every following byte
+    ``[continue][7 value bits]``, least significant group first.
+
+    Parameters
+    ----------
+    buf : bytes
+        Buffer to read from.
+    pos : int
+        Position of the varint.
+
+    Returns
+    -------
+    value, position : int
+        The value and the position after it.
+    """
+    byte = buf[pos]
+    pos += 1
+    negative = byte & 0x40
+    value = byte & 0x3F
+    shift = 6
+    while byte & 0x80:
+        byte = buf[pos]
+        pos += 1
+        value |= (byte & 0x7F) << shift
+        shift += 7
+    return (-value if negative else value), pos
+
+
+def decode_svarints(buf: bytes) -> tuple[np.ndarray, np.ndarray]:
+    """Decode a buffer of signed varints (vectorised).
+
+    Parameters
+    ----------
+    buf : bytes
+        Consecutive signed varints.
+
+    Returns
+    -------
+    values : numpy.ndarray
+        The values (int64).
+    end_markers : numpy.ndarray
+        Boolean mask of "negative zero" values, which end a block of points.
+
+    Raises
+    ------
+    CorruptFile
+        If the buffer ends inside a varint or a varint is too long.
+    """
+    data = np.frombuffer(buf, dtype=np.uint8)
+    if data.size == 0:
+        return np.empty(0, dtype=np.int64), np.empty(0, dtype=bool)
+    is_last = data < 0x80
+    if not is_last[-1]:
+        raise CorruptFile("Truncated varint at the end of a data block.")
+    ends = np.flatnonzero(is_last)
+    starts = np.concatenate(([0], ends[:-1] + 1))
+    lengths = ends - starts + 1
+    if lengths.max() > 9:
+        raise CorruptFile("Varint longer than 63 bits in data block.")
+    # the first byte of a varint carries 6 value bits, every following byte 7
+    pos_in_varint = np.arange(data.size) - np.repeat(starts, lengths)
+    shifts = np.where(pos_in_varint == 0, 0, 7 * pos_in_varint - 1)
+    bits = np.where(pos_in_varint == 0, data & 0x3F, data & 0x7F).astype(np.int64)
+    magnitude = np.add.reduceat(bits << shifts, starts)
+    negative = (data[starts] & 0x40) != 0
+    return np.where(negative, -magnitude, magnitude), negative & (magnitude == 0)
+
+
+def _iter_chunks(data: bytes) -> list[tuple[bytes, bytes]]:
+    """Split a raw signal file into (tag, payload) chunks."""
+    chunks = []
+    offset = 0
+    while offset < len(data):
+        if offset + _CHUNK_HEADER.size > len(data):
+            raise CorruptFile(f"Truncated chunk header at offset {offset}.")
+        tag, length = _CHUNK_HEADER.unpack_from(data, offset)
+        if length < _CHUNK_HEADER.size or offset + length > len(data):
+            raise CorruptFile(
+                f"Invalid length {length} for chunk {tag!r} at offset {offset}."
+            )
+        chunks.append((tag, data[offset + _CHUNK_HEADER.size : offset + length]))
+        offset += length
+    return chunks
+
+
+def _parse_signal_header(payload: bytes) -> dict:
+    """Parse the `SignHdr` chunk.
+
+    Fields with unknown meaning are skipped. A header that cannot be parsed
+    only costs metadata, never signal data.
+    """
+    try:
+        pos = 0
+        values = []
+        for _ in range(5):  # format version, unknown, major, minor, revision
+            value, pos = read_svarint(payload, pos)
+            values.append(value)
+        acquisition_start = filetime(struct.unpack_from("<q", payload, pos)[0])
+        pos += 8
+        _, pos = read_svarint(payload, pos)
+        name_length, pos = read_svarint(payload, pos)
+        channel = payload[pos : pos + name_length].decode("utf-8")
+        pos += name_length
+        injection_guid = uuid.UUID(bytes_le=payload[pos : pos + 16])
+        pos += 16
+        build, pos = read_svarint(payload, pos)
+    except (IndexError, OverflowError, struct.error, ValueError) as e:
+        warnings.warn(
+            f"Could not parse signal header: {e}", ChromparserWarning, stacklevel=3
+        )
+        return {}
+
+    major, minor, revision = values[2:5]
+    return {
+        "channel": channel,
+        "acquisition_start": acquisition_start,
+        "injection_guid": str(injection_guid),
+        "generating_data_system": f"Chromeleon {major}.{minor}.{revision}.{build}",
+    }
+
+
+def _decode_points_chunk(payload: bytes) -> tuple[np.ndarray, float, np.ndarray]:
+    """Decode a `PtsLL2Df` chunk.
+
+    Layout: two float64 scale factors (time ticks per minute, signal counts
+    per unit), then signed varints ``t0, v0, dt, dv, (ddt, ddv)...`` and a
+    "negative zero" end marker: a second-order delta encoding of the integer
+    time and signal.
+
+    Returns
+    -------
+    ticks : numpy.ndarray
+        Integer time (int64).
+    ticks_per_minute : float
+        Scale of `ticks`.
+    values : numpy.ndarray
+        Signal (float64) in the signal unit.
+    """
+    if len(payload) < 16:
+        raise CorruptFile("Truncated PtsLL2Df data block.")
+    ticks_per_minute, counts_per_unit = struct.unpack_from("<dd", payload, 0)
+    if not (ticks_per_minute > 0 and counts_per_unit > 0):
+        raise CorruptFile(
+            f"Invalid scale factors ({ticks_per_minute}, {counts_per_unit}) "
+            "in PtsLL2Df data block."
+        )
+    values, end_markers = decode_svarints(payload[16:])
+    end = np.flatnonzero(end_markers)
+    if end.size == 0:
+        raise CorruptFile("PtsLL2Df data block has no end marker.")
+    if end[0] != values.size - 1:
+        raise CorruptFile("Unexpected data after the end of a PtsLL2Df data block.")
+    values = values[: end[0]]
+
+    if values.size == 0:
+        return np.empty(0, dtype=np.int64), ticks_per_minute, np.empty(0)
+    if values.size % 2:
+        raise CorruptFile("PtsLL2Df data block has an incomplete data point.")
+    # rows: (t0, v0), (dt, dv), then (ddt, ddv) for every further point
+    pairs = values.reshape(-1, 2)
+    steps = np.cumsum(pairs[1:], axis=0)
+    points = np.vstack((pairs[:1], pairs[0] + np.cumsum(steps, axis=0)))
+    return points[:, 0], ticks_per_minute, points[:, 1] / counts_per_unit
+
+
+def decompress_lzma2(payload: bytes) -> bytes:
+    """Decompress a raw LZMA2 stream preceded by its dictionary-size byte.
+
+    Parameters
+    ----------
+    payload : bytes
+        Property byte, then the LZMA2 chunks.
+
+    Returns
+    -------
+    bytes
+        The decompressed data.
+
+    Raises
+    ------
+    CorruptFile
+        If the stream is empty or corrupt.
+    """
+    if not payload:
+        raise CorruptFile("Empty LZMA2 stream.")
+    prop = payload[0]
+    if prop > 40:
+        raise CorruptFile(f"Invalid LZMA2 dictionary size property {prop}.")
+    dict_size = 0xFFFFFFFF if prop == 40 else (2 | (prop & 1)) << (prop // 2 + 11)
+    filters = [{"id": lzma.FILTER_LZMA2, "dict_size": dict_size}]
+    try:
+        return lzma.LZMADecompressor(lzma.FORMAT_RAW, filters=filters).decompress(
+            payload[1:]
+        )
+    except lzma.LZMAError as e:
+        raise CorruptFile(f"Corrupt LZMA2 stream: {e}") from e
+
+
+def _decode_compressed_points_chunk(payload: bytes) -> tuple[np.ndarray, np.ndarray]:
+    """Decode a `PtsDDCmp` chunk into time (min) and signal arrays.
+
+    Layout: signed varints ``1, 0, uncompressed size, compressed size``, then
+    an LZMA2 stream of float64 ``(time, value)`` pairs.
+    """
+    try:
+        pos = 0
+        for _ in range(2):  # 1 and 0 in all known files, meaning unknown
+            _, pos = read_svarint(payload, pos)
+        size, pos = read_svarint(payload, pos)
+        compressed_size, pos = read_svarint(payload, pos)
+    except IndexError:
+        raise CorruptFile("Truncated PtsDDCmp data block.") from None
+    if compressed_size != len(payload) - pos:
+        raise CorruptFile(
+            f"PtsDDCmp data block has {len(payload) - pos} compressed bytes, "
+            f"its header says {compressed_size}."
+        )
+    data = decompress_lzma2(payload[pos:])
+    if len(data) != size or size % 16:
+        raise CorruptFile(
+            f"PtsDDCmp data block decompressed to {len(data)} bytes, "
+            f"expected {size} bytes of (time, value) pairs."
+        )
+    points = np.frombuffer(data, dtype="<f8").reshape(-1, 2)
+    return points[:, 0].copy(), points[:, 1].copy()
+
+
+def _regular_interval(ticks: list[np.ndarray], scales: list[float]) -> float | None:
+    """Return the sampling interval (s) of evenly spaced integer time ticks."""
+    if not ticks or len(set(scales)) != 1:
+        return None
+    steps = np.diff(np.concatenate(ticks))
+    if steps.size == 0 or np.any(steps != steps[0]) or steps[0] <= 0:
+        return None
+    return round(float(steps[0]) / scales[0] * 60, 12)
+
+
+def _float_interval(time: np.ndarray) -> float | None:
+    """Return the sampling interval (s) of an evenly spaced float time axis."""
+    steps = np.diff(time)
+    if steps.size == 0 or steps[0] <= 0 or np.ptp(steps) > 1e-6 * steps[0]:
+        return None
+    return round(float(time[-1] - time[0]) / (time.size - 1) * 60, 9)
+
+
+def decode_signal(data: bytes) -> Signal:
+    """Decode the bytes of a raw signal file.
+
+    Parameters
+    ----------
+    data : bytes
+        Content of the file.
+
+    Returns
+    -------
+    Signal
+        Without unit, which raw files do not store.
+
+    Raises
+    ------
+    UnsupportedFormat
+        If the data is not a raw signal file.
+    UnsupportedVariant
+        If it has an unknown chunk type.
+    CorruptFile
+        If it is malformed.
+    """
+    if not data.startswith(SIGNAL_MAGIC):
+        if data.startswith(AUDIT_TRAIL_MAGIC):
+            raise UnsupportedFormat(
+                "File is a Chromeleon audit trail (CpAu), not a signal. Signal "
+                "data is stored in the raw files starting with 'SignHdr'."
+            )
+        if data.startswith(THERMO_RAW_MAGIC):
+            raise UnsupportedFormat(
+                "File is a Thermo MS raw file, not a signal. Use read_thermo_raw "
+                "to read it."
+            )
+        raise UnsupportedFormat("Not a raw signal file (missing 'SignHdr' header).")
+
+    header: dict = {}
+    times, values = [], []
+    ticks, scales, compressed = [], [], False
+    for tag, payload in _iter_chunks(data):
+        if tag == SIGNAL_MAGIC:
+            header = _parse_signal_header(payload)
+        elif tag == POINTS_TAG:
+            tick, scale, value = _decode_points_chunk(payload)
+            ticks.append(tick)
+            scales.append(scale)
+            times.append(tick / scale)
+            values.append(value)
+        elif tag == COMPRESSED_POINTS_TAG:
+            time, value = _decode_compressed_points_chunk(payload)
+            compressed = True
+            times.append(time)
+            values.append(value)
+        else:
+            raise UnsupportedVariant(f"Unsupported chunk {tag!r} in raw signal file.")
+
+    time = np.concatenate(times) if times else np.empty(0)
+    if compressed:
+        interval = None if ticks else _float_interval(time)
+    else:
+        interval = _regular_interval(ticks, scales)
+    return Signal(
+        channel=header.pop("channel", ""),
+        time=time,
+        values=np.concatenate(values) if values else np.empty(0),
+        acquisition_start=header.pop("acquisition_start", None),
+        metadata=header,
+        sampling_interval=interval,
+    )
+
+
+def read_signal(source: Source, signal_unit: str | None = None) -> Signal:
+    """Read a raw signal file into a `Signal`.
+
+    This is cmbx-reader's ``read_raw``. A raw file stores neither the signal
+    unit nor the instrument's time zone; `read_cmbx` gets both from an archive.
+
+    Parameters
+    ----------
+    source : str, Path, bytes or file object
+        The raw signal file.
+    signal_unit : str, optional
+        Signal unit to attach, e.g. ``"pA"``.
+
+    Returns
+    -------
+    Signal
+        With `acquisition_start` in UTC.
+    """
+    signal = decode_signal(read_bytes(source))
+    signal.unit = signal_unit
+    return signal
+
+
+def read_chromeleon_raw(
+    path: str | Path, *, signal_unit: str | None = None
+) -> list[Injection]:
+    """Read a Chromeleon raw signal file.
+
+    Parameters
+    ----------
+    path : str or Path
+        The raw signal file.
+    signal_unit : str, optional
+        Signal unit, which raw files do not store.
+
+    Returns
+    -------
+    list of Injection
+        One injection without name and inject time, holding the signal.
+    """
+    signal = read_signal(existing_file(path), signal_unit)
+    guid = signal.metadata.get("injection_guid")
+    return [
+        Injection(
+            name="",
+            signals={signal.channel: signal},
+            metadata={"guid": guid} if guid else {},
+        )
+    ]
+
+
+def sniff_chromeleon_raw(path: str | Path) -> bool:
+    """Whether `path` is a Chromeleon raw signal file (starts with `SignHdr`)."""
+    return Path(path).is_file() and head(path, 8) == SIGNAL_MAGIC

chromparser/vendors/chromeleon/sequence.py

@@ -0,0 +1,177 @@
+"""Metadata from the sequence file (`*.cmd`) of a `.cmbx` archive.
+
+The sequence is serialized with protobuf-net. Top-level field 18 holds item
+records (type name, own GUID, parent GUID), field 19 the data of each item,
+linked by GUID. Only the fields listed in `docs/formats/chromeleon.md` are read.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+from ..._util import clean as _clean
+from ..._util import parse_iso_datetime
+from ...errors import CorruptFile
+
+
+@dataclass
+class SignalInfo:
+    """Signal metadata from the sequence, with the metadata of its injection."""
+
+    unit: str | None = None
+    expected_points: int | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    injection_name: str | None = None
+    inject_time: datetime | None = None
+    injection_metadata: dict[str, Any] = field(default_factory=dict)
+
+
+def _read_uvarint(buf: bytes, pos: int) -> tuple[int, int]:
+    result = shift = 0
+    while True:
+        byte = buf[pos]
+        pos += 1
+        result |= (byte & 0x7F) << shift
+        shift += 7
+        if not byte & 0x80:
+            return result, pos
+
+
+def _decode(buf: bytes) -> dict[int, list]:
+    """Decode a protobuf message into {field number: [raw values]}."""
+    fields: dict[int, list] = {}
+    pos = 0
+    while pos < len(buf):
+        key, pos = _read_uvarint(buf, pos)
+        wire_type = key & 7
+        if wire_type == 0:
+            value, pos = _read_uvarint(buf, pos)
+        elif wire_type == 1:
+            value, pos = buf[pos : pos + 8], pos + 8
+        elif wire_type == 2:
+            length, pos = _read_uvarint(buf, pos)
+            value, pos = buf[pos : pos + length], pos + length
+        elif wire_type == 5:
+            value, pos = buf[pos : pos + 4], pos + 4
+        else:
+            raise CorruptFile(f"Unsupported protobuf wire type {wire_type}.")
+        if pos > len(buf):
+            raise CorruptFile("Truncated protobuf message.")
+        fields.setdefault(key >> 3, []).append(value)
+    return fields
+
+
+def _get(message: dict, *path: int):
+    """Return the first value at a path of nested field numbers, or None."""
+    for depth, field_number in enumerate(path):
+        values = message.get(field_number)
+        if not values:
+            return None
+        if depth == len(path) - 1:
+            return values[0]
+        message = _decode(values[0])
+    return None
+
+
+def _str(message: dict, *path: int) -> str | None:
+    value = _get(message, *path)
+    return value.decode("utf-8") if isinstance(value, bytes) else None
+
+
+def _guid(message: dict, field_number: int) -> uuid.UUID | None:
+    """Decode a protobuf-net `bcl.Guid` (two fixed64 halves)."""
+    value = _get(message, field_number)
+    if not isinstance(value, bytes):
+        return None
+    halves = _decode(value)
+    low = halves.get(1, [b"\x00" * 8])[0]
+    high = halves.get(2, [b"\x00" * 8])[0]
+    return uuid.UUID(bytes_le=low + high)
+
+
+_PROCESSING_METHOD = 0
+_INSTRUMENT_METHOD = 1
+
+
+def _method_names(injection: dict) -> dict[int, str]:
+    """Names of the methods linked in field 40, by kind (field 40.1)."""
+    names = {}
+    for raw in injection.get(40, []):
+        link = _decode(raw)
+        kind = _get(link, 1)
+        name = _str(link, 2, 5)
+        if name:
+            names.setdefault(kind if isinstance(kind, int) else 0, name)
+    return names
+
+
+def _signal_info(record: dict, injection: dict | None) -> SignalInfo:
+    signal = _decode(_get(record, 5, 3) or b"")
+    expected_points = _get(signal, 4)
+    info = SignalInfo(
+        unit=_str(signal, 6, 3) or None,
+        expected_points=expected_points if isinstance(expected_points, int) else None,
+        metadata=_clean(
+            {
+                "signal_quantity": _str(signal, 6, 4),
+                "detector": _str(signal, 11),
+                "driver": _str(signal, 12),
+            }
+        ),
+    )
+    if injection is not None:
+        info.injection_name = _str(injection, 19, 2)
+        inject_time = _str(injection, 19, 7, 1)
+        info.inject_time = parse_iso_datetime(inject_time) if inject_time else None
+        methods = _method_names(injection)
+        info.injection_metadata = _clean(
+            {
+                "type": _str(injection, 19, 3, 3),
+                "status": _str(injection, 19, 4, 3),
+                "position": _str(injection, 19, 5),
+                "instrument_method": methods.get(_INSTRUMENT_METHOD),
+                "processing_method": methods.get(_PROCESSING_METHOD),
+            }
+        )
+    return info
+
+
+def parse_sequence(data: bytes) -> dict[str, SignalInfo]:
+    r"""Extract signal and injection metadata from a sequence file.
+
+    Parameters
+    ----------
+    data : bytes
+        Content of the `*.cmd` file.
+
+    Returns
+    -------
+    dict of str to SignalInfo
+        Mapping of raw data file id (lower case, e.g.
+        ``2023\\09\\07\\152612018.raw``) to the metadata of that signal.
+    """
+    top = _decode(data)
+    items: dict[uuid.UUID, tuple[str | None, uuid.UUID | None]] = {}
+    for raw in top.get(18, []):
+        record = _decode(raw)
+        guid = _guid(record, 5)
+        if guid is not None:
+            items[guid] = (_str(record, 3, 3), _guid(record, 6))
+    payloads = {}
+    for raw in top.get(19, []):
+        record = _decode(raw)
+        guid = _guid(record, 25)
+        if guid is not None:
+            payloads[guid] = record
+
+    signals = {}
+    for guid, (item_type, parent) in items.items():
+        if item_type != "Signal" or guid not in payloads:
+            continue
+        raw_id = _str(payloads[guid], 5, 6)
+        if raw_id:
+            signals[raw_id.lower()] = _signal_info(payloads[guid], payloads.get(parent))
+    return signals