natus-erd-reader 0.2.1__py3-none-any.whl

natus_erd/__init__.py

@@ -0,0 +1,26 @@
+"""Lazy native reader for Natus NeuroWorks ERD recordings."""
+
+from .errors import (
+    DataIntegrityError,
+    NatusERDError,
+    ResourceLimitError,
+    UnsupportedFormatError,
+)
+from .limits import ReadLimits
+from .models import ChannelInfo, Event, RecordingInfo, ValidationReport
+from .reader import NatusERDReader
+
+__all__ = [
+    "ChannelInfo",
+    "DataIntegrityError",
+    "Event",
+    "NatusERDError",
+    "NatusERDReader",
+    "RecordingInfo",
+    "ReadLimits",
+    "ResourceLimitError",
+    "UnsupportedFormatError",
+    "ValidationReport",
+]
+
+__version__ = "0.2.1"

natus_erd/_paths.py

@@ -0,0 +1,92 @@
+"""Bounded, non-recursive lookup of the real names in a recording directory."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from .binary import regular_file_size
+from .errors import DataIntegrityError, UnsupportedFormatError
+from .limits import DEFAULT_LIMITS, ReadLimits, check_limit
+
+
+class RecordingDirectory:
+    """Snapshot directory names, not payloads; resolve only selected members.
+
+    Windows recordings often use mixed-case extensions. Use unique casefold
+    matches on every platform without guessing between colliding filenames.
+    The top-level path is still resolved by the operating system as supplied.
+    """
+
+    def __init__(self, directory: Path, *, limits: ReadLimits = DEFAULT_LIMITS) -> None:
+        self.directory = directory.resolve(strict=True)
+        if any(part.casefold() == "decimated" for part in self.directory.parts):
+            raise UnsupportedFormatError("The Decimated derivative is outside the supported scope")
+        self._names: dict[str, list[Path]] = {}
+        self._stc_names: list[str] = []
+        with os.scandir(self.directory) as entries:
+            for count, entry in enumerate(entries, 1):
+                check_limit(count, limits.max_directory_entries, "Recording directory entries")
+                self._names.setdefault(entry.name.casefold(), []).append(Path(entry.path))
+                # An unrelated ordinary backup.stc/ folder is not a recording
+                # candidate. Links remain candidates so boundary checks apply.
+                if entry.name.casefold().endswith(".stc") and not entry.is_dir(follow_symlinks=False):
+                    self._stc_names.append(entry.name)
+
+    def lookup(self, name: str, *, optional: bool = False) -> Path | None:
+        matches = self._names.get(name.casefold(), ())
+        if len(matches) > 1:
+            raise DataIntegrityError("Ambiguous case-insensitive recording filename")
+        if not matches:
+            if optional:
+                return None
+            raise DataIntegrityError("Missing required recording file")
+        try:
+            resolved = matches[0].resolve(strict=True)
+        except (OSError, RuntimeError) as exc:
+            raise DataIntegrityError("Missing or inaccessible recording file") from exc
+        if resolved.parent != self.directory:
+            raise DataIntegrityError("Recording file resolves outside its directory")
+        regular_file_size(resolved)
+        return resolved
+
+    def stc_paths(self) -> tuple[Path, ...]:
+        paths: list[Path] = []
+        for name in self._stc_names:
+            path = self.lookup(name)
+            assert path is not None  # Required lookup raises instead of returning None.
+            paths.append(path)
+        return tuple(paths)
+
+    def resolve_stc(self, source: Path) -> Path:
+        """Select one recording without scanning other records' metadata."""
+        if source.is_file():
+            suffix = source.suffix.casefold()
+            if suffix not in {".stc", ".eeg", ".erd"}:
+                raise ValueError("Expected a recording directory or EEG/STC/ERD file")
+            self.lookup(source.name)  # Also reject case collisions on an explicit entry.
+            if suffix == ".stc":
+                return source
+            if suffix == ".eeg":
+                stc = self.lookup(source.stem + ".stc", optional=True)
+                if stc is None:
+                    raise FileNotFoundError("No STC matches the supplied EEG file")
+                return stc
+        elif not source.is_dir():
+            raise ValueError("Expected a recording directory or EEG/STC/ERD file")
+
+        candidates = self.stc_paths()
+        if not candidates:
+            raise FileNotFoundError("No STC file found; pass the recording directory itself (discovery is not recursive)")
+        if len(candidates) != 1:
+            raise DataIntegrityError(
+                f"Expected one main STC file, found {len(candidates)}; pass the target STC or matching EEG file"
+            )
+        return candidates[0]
+
+
+def resolve_recording(path: Path, *, limits: ReadLimits = DEFAULT_LIMITS) -> tuple[Path, RecordingDirectory]:
+    source = path.expanduser().resolve(strict=True)
+    directory = source if source.is_dir() else source.parent
+    files = RecordingDirectory(directory, limits=limits)
+    return files.resolve_stc(source), files

natus_erd/binary.py

@@ -0,0 +1,292 @@
+"""Binary structure parsing for NeuroWorks schema-9 recordings."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from math import isfinite
+import os
+from pathlib import Path
+import stat
+from struct import Struct, unpack_from
+
+from .errors import DataIntegrityError, UnsupportedFormatError
+from .limits import DEFAULT_LIMITS, ReadLimits, check_limit
+
+GENERIC_HEADER_SIZE = 352
+ERD_HEADER_SIZE = 8656
+STC_PREFIX_SIZE = 408
+STC_ENTRY_SIZE = 272
+ETC_ENTRY_SIZE = 16
+
+
+@dataclass(frozen=True, slots=True)
+class GenericHeader:
+    file_schema: int
+    base_schema: int
+    creation_time_unix: int
+
+
+@dataclass(frozen=True, slots=True)
+class ErdHeader:
+    generic: GenericHeader
+    sample_rate: float
+    n_channels: int
+    delta_bits: int
+    physical_channels: tuple[int, ...]
+    headbox_types: tuple[int, int, int, int]
+    discard_bits: int
+    shorted: tuple[bool, ...]
+    frequency_factors: tuple[int, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class StcEntry:
+    index: int
+    segment_name: str
+    start_stamp: int
+    end_stamp: int
+    sample_number: int
+    stored_samples: int
+
+    @property
+    def logical_span(self) -> int:
+        """Elapsed sample positions, including gaps with no stored packets."""
+        return self.end_stamp - self.start_stamp + 1
+
+
+@dataclass(frozen=True, slots=True)
+class StcFile:
+    generic: GenericHeader
+    next_segment: int
+    final: int
+    entries: tuple[StcEntry, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class EtcEntry:
+    index: int
+    offset: int
+    sample_stamp: int
+    sample_number: int
+    sample_span: int
+    unknown: int
+
+    @property
+    def end_stamp_exclusive(self) -> int:
+        return self.sample_stamp + self.sample_span
+
+
+def regular_file_size(path: Path) -> int:
+    """Reject special files before opening them (e.g. named pipes/devices)."""
+    try:
+        info = path.stat()
+    except OSError as exc:
+        raise DataIntegrityError(f"Cannot stat {path.name}: {exc}") from exc
+    if not stat.S_ISREG(info.st_mode):
+        raise DataIntegrityError(f"Expected a regular file: {path.name}")
+    return info.st_size
+
+
+def _read_file(path: Path, *, maximum: int) -> bytes:
+    """Check size before allocation and use a finite read even if a file grows."""
+    size = regular_file_size(path)
+    check_limit(size, maximum, "Metadata file bytes")
+    try:
+        with path.open("rb") as stream:
+            if os.fstat(stream.fileno()).st_size != size:
+                raise DataIntegrityError("Metadata changed while opening")
+            data = stream.read(size)
+            if len(data) != size or stream.read(1):
+                raise DataIntegrityError("Metadata changed or was truncated while reading")
+    except OSError as exc:
+        raise DataIntegrityError(f"Cannot read {path.name}: {exc}") from exc
+    return data
+
+
+def check_generic_schema(header: GenericHeader, expected: int, context: str) -> None:
+    if header.file_schema != expected or header.base_schema != 1:
+        raise UnsupportedFormatError(
+            f"Unsupported {context} schema {header.file_schema}, base {header.base_schema}"
+        )
+
+
+def _safe_segment_name(name: str) -> bool:
+    # Windows device/drive/alternate-stream names are unsafe even on POSIX.
+    if not name or name in {".", ".."} or name[-1] in " .":
+        return False
+    if any(char in '<>:"/\\|?*' or ord(char) < 32 for char in name):
+        return False
+    device = name.split(".", 1)[0].upper()
+    return device not in {"CON", "PRN", "AUX", "NUL", "CLOCK$", "CONIN$", "CONOUT$"} and not (
+        len(device) == 4 and device[:3] in {"COM", "LPT"}
+        and device[3] in "123456789\u00b9\u00b2\u00b3"
+    )
+
+
+def _generic_from_bytes(data: bytes, path: Path) -> GenericHeader:
+    if len(data) < GENERIC_HEADER_SIZE:
+        raise DataIntegrityError(
+            f"{path.name} is shorter than the {GENERIC_HEADER_SIZE}-byte header"
+        )
+    file_schema, base_schema = unpack_from("<HH", data, 16)
+    creation_time = unpack_from("<i", data, 20)[0]
+    return GenericHeader(file_schema, base_schema, creation_time)
+
+
+def read_generic_header(path: Path) -> GenericHeader:
+    regular_file_size(path)
+    try:
+        with path.open("rb") as stream:
+            data = stream.read(GENERIC_HEADER_SIZE)
+    except OSError as exc:
+        raise DataIntegrityError(f"Cannot read {path.name}: {exc}") from exc
+    return _generic_from_bytes(data, path)
+
+
+def read_erd_header(path: Path) -> ErdHeader:
+    regular_file_size(path)
+    try:
+        with path.open("rb") as stream:
+            data = stream.read(ERD_HEADER_SIZE)
+    except OSError as exc:
+        raise DataIntegrityError(f"Cannot read {path.name}: {exc}") from exc
+
+    if len(data) < ERD_HEADER_SIZE:
+        raise DataIntegrityError(
+            f"{path.name} is shorter than the {ERD_HEADER_SIZE}-byte ERD header"
+        )
+    generic = _generic_from_bytes(data, path)
+    check_generic_schema(generic, 9, "ERD")
+    sample_rate = unpack_from("<d", data, 352)[0]
+    n_channels, delta_bits = unpack_from("<ii", data, 360)
+    if not 1 <= n_channels <= 1024:
+        raise DataIntegrityError(
+            f"{path.name} declares an invalid channel count: {n_channels}"
+        )
+    if not isfinite(sample_rate) or sample_rate <= 0:
+        raise DataIntegrityError("ERD sample rate must be finite and positive")
+    physical = unpack_from(f"<{n_channels}i", data, 368)
+    headbox_values = unpack_from("<4i", data, 4464)
+    headbox_types: tuple[int, int, int, int] = (
+        int(headbox_values[0]),
+        int(headbox_values[1]),
+        int(headbox_values[2]),
+        int(headbox_values[3]),
+    )
+    discard_bits = unpack_from("<i", data, 4556)[0]
+    shorted_raw = unpack_from("<1024h", data, 4560)[:n_channels]
+    if any(value not in (0, 1) for value in shorted_raw):
+        raise DataIntegrityError("ERD shorted flags must be zero or one")
+    frequency_factors = unpack_from("<1024h", data, 6608)[:n_channels]
+    return ErdHeader(
+        generic=generic,
+        sample_rate=sample_rate,
+        n_channels=n_channels,
+        delta_bits=delta_bits,
+        physical_channels=tuple(physical),
+        headbox_types=headbox_types,
+        discard_bits=discard_bits,
+        shorted=tuple(bool(value) for value in shorted_raw),
+        frequency_factors=tuple(frequency_factors),
+    )
+
+
+def read_stc(path: Path, *, limits: ReadLimits = DEFAULT_LIMITS) -> StcFile:
+    data = _read_file(
+        path, maximum=min(limits.max_metadata_bytes, STC_PREFIX_SIZE + limits.max_segments * STC_ENTRY_SIZE)
+    )
+    generic = _generic_from_bytes(data, path)
+    check_generic_schema(generic, 1, "STC")
+    if len(data) < STC_PREFIX_SIZE:
+        raise DataIntegrityError(f"{path.name} has a truncated STC prefix")
+    payload_size = len(data) - STC_PREFIX_SIZE
+    if payload_size % STC_ENTRY_SIZE:
+        raise DataIntegrityError(
+            f"{path.name} has {payload_size % STC_ENTRY_SIZE} trailing STC bytes"
+        )
+
+    next_segment, final = unpack_from("<ii", data, GENERIC_HEADER_SIZE)
+    entries: list[StcEntry] = []
+    names: set[str] = set()
+    for index, offset in enumerate(
+        range(STC_PREFIX_SIZE, len(data), STC_ENTRY_SIZE)
+    ):
+        raw_name = data[offset : offset + 256].split(b"\0", 1)[0]
+        try:
+            name = raw_name.decode("utf-8")
+        except UnicodeDecodeError as exc:
+            raise DataIntegrityError(
+                f"STC segment {index} has an invalid UTF-8 name"
+            ) from exc
+        if not _safe_segment_name(name):
+            raise DataIntegrityError(f"STC segment {index} has an unsafe name")
+        stem = name[:-4] if name.casefold().endswith(".erd") else name
+        if not _safe_segment_name(stem) or stem.casefold() in names:
+            raise DataIntegrityError(f"STC segment {index} has an unsafe or duplicate stem")
+        names.add(stem.casefold())
+        start, end, sample_number, stored_samples = unpack_from("<4i", data, offset + 256)
+        if end < start:
+            raise DataIntegrityError(
+                f"STC segment {index} has inconsistent stamp bounds"
+            )
+        if not 0 <= stored_samples <= end - start + 1:
+            raise DataIntegrityError(f"STC segment {index} has an invalid stored sample count")
+        if entries and start <= entries[-1].end_stamp:
+            raise DataIntegrityError(f"STC segments overlap at entry {index}")
+        entries.append(
+            StcEntry(index, name, start, end, sample_number, stored_samples)
+        )
+
+    if not entries:
+        raise DataIntegrityError(f"{path.name} contains no STC entries")
+    return StcFile(generic, next_segment, final, tuple(entries))
+
+
+_ETC_STRUCT = Struct("<iiihh")
+
+
+def read_etc(
+    path: Path, *, erd_size: int | None = None, limits: ReadLimits = DEFAULT_LIMITS
+) -> tuple[EtcEntry, ...]:
+    data = _read_file(
+        path, maximum=min(limits.max_metadata_bytes, GENERIC_HEADER_SIZE + limits.max_packets_per_segment * ETC_ENTRY_SIZE)
+    )
+    check_generic_schema(_generic_from_bytes(data, path), 3, "ETC")
+    payload_size = len(data) - GENERIC_HEADER_SIZE
+    if payload_size % ETC_ENTRY_SIZE:
+        raise DataIntegrityError(
+            f"{path.name} has {payload_size % ETC_ENTRY_SIZE} trailing ETC bytes"
+        )
+
+    entries: list[EtcEntry] = []
+    for index, offset in enumerate(
+        range(GENERIC_HEADER_SIZE, len(data), ETC_ENTRY_SIZE)
+    ):
+        data_offset, stamp, sample_number, span, unknown = _ETC_STRUCT.unpack_from(
+            data, offset
+        )
+        if span <= 0:
+            raise DataIntegrityError(f"ETC packet {index} has invalid span {span}")
+        if data_offset < ERD_HEADER_SIZE:
+            raise DataIntegrityError(
+                f"ETC packet {index} points inside the ERD header"
+            )
+        if erd_size is not None and data_offset >= erd_size:
+            raise DataIntegrityError(
+                f"ETC packet {index} offset is outside its ERD file"
+            )
+        if entries:
+            previous = entries[-1]
+            if data_offset <= previous.offset:
+                raise DataIntegrityError(f"ETC offsets are not increasing at {index}")
+            if stamp < previous.end_stamp_exclusive:
+                raise DataIntegrityError(f"ETC packets overlap at entry {index}")
+            expected_number = previous.sample_number + previous.sample_span
+            if sample_number != expected_number:
+                raise DataIntegrityError(
+                    f"ETC sample numbers are discontinuous at entry {index}"
+                )
+        entries.append(
+            EtcEntry(index, data_offset, stamp, sample_number, span, unknown)
+        )
+    return tuple(entries)

natus_erd/decoder.py

@@ -0,0 +1,207 @@
+"""Bounded, incremental decoder for schema-9 compressed ERD packets."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from numbers import Integral
+from struct import unpack_from
+from typing import BinaryIO, TYPE_CHECKING
+
+from .errors import DataIntegrityError
+from .limits import DEFAULT_LIMITS, ReadLimits, check_limit, check_output_size
+
+if TYPE_CHECKING:
+    import numpy as np
+    from numpy.typing import NDArray
+
+
+_READ_CHUNK_BYTES = 64 * 1024
+
+
+def _integer(value: int, context: str) -> int:
+    if isinstance(value, bool) or not isinstance(value, Integral):
+        raise ValueError(f"{context} must be an integer")
+    return int(value)
+
+
+def validate_packet_bounds(
+    *,
+    offset: int,
+    byte_end: int,
+    sample_count: int,
+    n_channels: int,
+    shorted: Sequence[bool],
+    limits: ReadLimits = DEFAULT_LIMITS,
+) -> None:
+    """Reject impossible or oversized packet spans without reading any bytes."""
+    offset = _integer(offset, "packet offset")
+    byte_end = _integer(byte_end, "packet end")
+    sample_count = _integer(sample_count, "packet sample count")
+    n_channels = _integer(n_channels, "packet channel count")
+    if offset < 0 or byte_end <= offset:
+        raise DataIntegrityError("ERD packet has an invalid byte range")
+    if not 1 <= sample_count <= 32767:
+        raise DataIntegrityError("ERD packet sample count must be between 1 and 32767")
+    if not 1 <= n_channels <= 1024:
+        raise DataIntegrityError("ERD packet channel count must be between 1 and 1024")
+    if len(shorted) != n_channels or any(value not in (False, True) for value in shorted):
+        raise DataIntegrityError("ERD shorted mask must contain one boolean per channel")
+
+    mask_bytes = (n_channels + 7) // 8
+    active = sum(not value for value in shorted)
+    maximum_sample_bytes = 1 + mask_bytes + 6 * active
+    minimum_bytes = maximum_sample_bytes + (sample_count - 1) * (1 + mask_bytes + active)
+    maximum_bytes = sample_count * maximum_sample_bytes
+    packet_bytes = byte_end - offset
+    if not minimum_bytes <= packet_bytes <= maximum_bytes:
+        raise DataIntegrityError(
+            "ERD packet byte span is impossible for its sample and channel counts"
+        )
+    check_limit(packet_bytes, limits.max_packet_bytes, "Compressed ERD packet bytes")
+
+
+def decode_schema9_packet(
+    stream: BinaryIO,
+    *,
+    offset: int,
+    byte_end: int,
+    sample_count: int,
+    start: int,
+    stop: int,
+    n_channels: int,
+    shorted: Sequence[bool],
+    selected: Sequence[int],
+    limits: ReadLimits = DEFAULT_LIMITS,
+    out: NDArray[np.float64] | None = None,
+) -> NDArray[np.float64]:
+    """Decode a half-open sample window from an independently compressed packet.
+
+    Byte ranges and allocation budgets are checked before NumPy is imported.
+    Reads are at most 64 KiB; the compressed working buffer is bounded by a
+    chunk plus one sample. Only the prefix needed to reach stop is decoded.
+    A complete-packet read additionally checks exact payload consumption.
+    out permits writing directly into a caller-provided float64 array view.
+    """
+    validate_packet_bounds(
+        offset=offset, byte_end=byte_end, sample_count=sample_count,
+        n_channels=n_channels, shorted=shorted, limits=limits,
+    )
+    offset, byte_end = int(offset), int(byte_end)
+    sample_count, n_channels = int(sample_count), int(n_channels)
+    start, stop = _integer(start, "packet start"), _integer(stop, "packet stop")
+    if not 0 <= start <= stop <= sample_count:
+        raise ValueError("packet sample bounds are invalid")
+    check_output_size(len(selected), stop - start, limits)
+    selected_rows: list[tuple[int, int]] = []
+    for row, raw_channel in enumerate(selected):
+        channel = _integer(raw_channel, "selected channel")
+        if not 0 <= channel < n_channels:
+            raise ValueError(f"channel index is out of range: {channel}")
+        if not shorted[channel]:
+            selected_rows.append((row, channel))
+
+    import numpy as np
+
+    shape = (len(selected), stop - start)
+    if out is None:
+        output = np.full(shape, np.nan, dtype=np.float64)
+    else:
+        if not isinstance(out, np.ndarray) or out.shape != shape or out.dtype != np.dtype("float64"):
+            raise ValueError("out must be a float64 NumPy array with the requested shape")
+        if not out.flags.writeable:
+            raise ValueError("out must be writeable")
+        output = out
+        output.fill(np.nan)
+    if start == stop:
+        return output
+
+    mask_size = (n_channels + 7) // 8
+    active_layout = tuple(
+        (channel, channel >> 3, 1 << (channel & 7))
+        for channel in range(n_channels)
+        if not shorted[channel]
+    )
+    maximum_sample_bytes = 1 + mask_size + 6 * len(active_layout)
+    packet_bytes = byte_end - offset
+    # Limiting the prefix avoids reading an entire large packet for a tiny window.
+    read_limit = min(packet_bytes, stop * maximum_sample_bytes)
+    state = [0] * n_channels
+    payload = b""
+    position = 0
+    consumed_before_buffer = 0
+    bytes_read = 0
+    payload_length = 0
+    try:
+        stream.seek(offset)
+    except OSError as exc:
+        raise DataIntegrityError(f"Cannot seek to ERD packet: {exc}") from exc
+
+    def require(size: int, context: str) -> None:
+        if position + size > payload_length:
+            raise DataIntegrityError(
+                f"ERD packet ended while reading {context} at sample {sample_index}"
+            )
+
+    for sample_index in range(stop):
+        # Refill only between samples, leaving the tight channel loop free of I/O.
+        if payload_length - position < maximum_sample_bytes and bytes_read < read_limit:
+            tail = payload[position:]
+            consumed_before_buffer += position
+            position = 0
+            requested = min(_READ_CHUNK_BYTES, read_limit - bytes_read)
+            try:
+                block = stream.read(requested)
+            except OSError as exc:
+                raise DataIntegrityError(f"Cannot read ERD packet: {exc}") from exc
+            if len(block) != requested:
+                raise DataIntegrityError("ERD packet is truncated")
+            bytes_read += len(block)
+            payload = tail + block
+            payload_length = len(payload)
+
+        require(1 + mask_size, "event byte and delta mask")
+        event = payload[position]
+        position += 1
+        if event not in (0, 1):
+            raise DataIntegrityError(
+                f"Invalid ERD event byte 0x{event:02x} at sample {sample_index}"
+            )
+        mask_position = position
+        position += mask_size
+
+        absolute_channels: list[int] = []
+        for channel, mask_byte, mask_bit in active_layout:
+            wide = bool(payload[mask_position + mask_byte] & mask_bit)
+            if wide:
+                require(2, "16-bit delta")
+                value = payload[position] | (payload[position + 1] << 8)
+                position += 2
+                if value >= 0x8000:
+                    value -= 0x10000
+                if value == -1:
+                    absolute_channels.append(channel)
+                    continue
+            else:
+                require(1, "8-bit delta")
+                value = payload[position]
+                position += 1
+                if value >= 0x80:
+                    value -= 0x100
+            if sample_index == 0:
+                raise DataIntegrityError("The first sample of an ERD packet is not absolute")
+            state[channel] += value
+
+        for channel in absolute_channels:
+            require(4, "absolute channel value")
+            state[channel] = unpack_from("<i", payload, position)[0]
+            position += 4
+        if sample_index == 0 and len(absolute_channels) != len(active_layout):
+            raise DataIntegrityError("The first ERD sample does not initialize every channel")
+        if sample_index >= start:
+            output_column = sample_index - start
+            for row, channel in selected_rows:
+                output[row, output_column] = state[channel]
+
+    if stop == sample_count and consumed_before_buffer + position != packet_bytes:
+        raise DataIntegrityError("ERD packet has unexplained trailing bytes")
+    return output