chromparser 0.1.0__py3-none-any.whl

chromparser/_netcdf3.py

@@ -0,0 +1,548 @@
+r"""Minimal netCDF-3 reader and writer (no runtime dependency).
+
+Implements Unidata's "netCDF classic and 64-bit offset format" specification:
+read classic (``CDF\x01``) and 64-bit offset (``CDF\x02``) files with fixed
+and record (unlimited) dimensions; write classic files. All values are
+big-endian; arrays are returned in native byte order with the stored type
+(``int8``, ``S1`` for chars, ``int16``, ``int32``, ``float32``, ``float64``).
+CDF-5 (``CDF\x05``) and netCDF-4 (HDF5) files are not read.
+"""
+
+from __future__ import annotations
+
+import struct
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, BinaryIO
+
+import numpy as np
+
+from .errors import CorruptFile, UnsupportedFormat, UnsupportedVariant
+
+CLASSIC = b"CDF\x01"
+OFFSET64 = b"CDF\x02"
+CDF5 = b"CDF\x05"
+STREAMING = 0xFFFFFFFF  # numrecs of a file still being written
+
+_DIMENSION, _VARIABLE, _ATTRIBUTE = 0x0A, 0x0B, 0x0C
+#: nc_type -> big-endian numpy dtype
+TYPES = {1: ">i1", 2: "S1", 3: ">i2", 4: ">i4", 5: ">f4", 6: ">f8"}
+_TYPE_OF = {np.dtype(v).newbyteorder("=").str[1:]: k for k, v in TYPES.items()}
+_TYPE_OF["S1"] = 2
+
+#: Default fill values of the netCDF library (used for unwritten data).
+FILL = {
+    1: -127,
+    2: 0,
+    3: -32767,
+    4: -2147483647,
+    5: 9.969209968386869e36,
+    6: 9.969209968386869e36,
+}
+
+
+def _pad(n: int) -> int:
+    return -n % 4
+
+
+def is_netcdf3(head: bytes) -> bool:
+    """Whether `head` (the first bytes of a file) is a netCDF-3 magic."""
+    return head[:4] in (CLASSIC, OFFSET64)
+
+
+# ------------------------------------------------------------------ reading
+class _Short(Exception):
+    """The header continues past the bytes read so far."""
+
+
+@dataclass
+class Variable:
+    """A variable of a netCDF-3 file, without its data (see `File.read`).
+
+    Attributes
+    ----------
+    name : str
+        Variable name.
+    dimensions : tuple of str
+        Dimension names; a record variable's first dimension is unlimited.
+    shape : tuple of int
+        Shape; the record dimension counts the file's records.
+    nc_type : int
+        netCDF type code (1 byte, 2 char, 3 short, 4 int, 5 float, 6 double).
+    attributes : dict
+        Attributes: `str` for chars, numpy scalars or 1D arrays otherwise.
+    begin : int
+        File offset of the data (of the first record for record variables).
+    is_record : bool
+        Whether the first dimension is the unlimited one.
+    """
+
+    name: str
+    dimensions: tuple[str, ...]
+    shape: tuple[int, ...]
+    nc_type: int
+    attributes: dict[str, Any] = field(default_factory=dict)
+    begin: int = 0
+    is_record: bool = False
+
+    @property
+    def dtype(self) -> np.dtype:
+        """Stored dtype in native byte order."""
+        return np.dtype(TYPES[self.nc_type]).newbyteorder("=")
+
+
+class _Parser:
+    def __init__(self, data: bytes, offset64: bool):
+        self.data = data
+        self.pos = 0
+        self.offset64 = offset64
+
+    def take(self, n: int) -> bytes:
+        if self.pos + n > len(self.data):
+            raise _Short
+        chunk = self.data[self.pos : self.pos + n]
+        self.pos += n
+        return chunk
+
+    def uint(self) -> int:
+        return struct.unpack(">I", self.take(4))[0]
+
+    def offset(self) -> int:
+        if self.offset64:
+            return struct.unpack(">Q", self.take(8))[0]
+        return self.uint()
+
+    def count(self) -> int:
+        n = self.uint()
+        if n >= 1 << 31:
+            raise CorruptFile(f"negative count {n - (1 << 32)} in netCDF header")
+        return n
+
+    def name(self) -> str:
+        n = self.count()
+        raw = self.take(n)
+        self.take(_pad(n))
+        return raw.decode("utf-8", "replace")
+
+    def nc_type(self) -> int:
+        t = self.uint()
+        if t not in TYPES:
+            raise UnsupportedVariant(f"netCDF type {t} (only netCDF-3 types are read)")
+        return t
+
+    def values(self, nc_type: int, n: int) -> Any:
+        dtype = np.dtype(TYPES[nc_type])
+        raw = self.take(n * dtype.itemsize)
+        self.take(_pad(n * dtype.itemsize))
+        if nc_type == 2:
+            return raw.split(b"\0", 1)[0].decode("utf-8", "replace")
+        values = np.frombuffer(raw, dtype).astype(dtype.newbyteorder("="))
+        return values[0] if n == 1 else values
+
+    def header_list(self, tag: int) -> int:
+        kind, n = self.uint(), self.count()
+        if kind == 0 and n == 0:
+            return 0
+        if kind != tag:
+            raise CorruptFile(f"netCDF header: expected list tag {tag}, found {kind}")
+        return n
+
+    def attributes(self) -> dict[str, Any]:
+        result = {}
+        for _ in range(self.header_list(_ATTRIBUTE)):
+            name = self.name()
+            nc_type = self.nc_type()
+            result[name] = self.values(nc_type, self.count())
+        return result
+
+
+class File:
+    """An open netCDF-3 file: dimensions, attributes and lazily read variables.
+
+    Use as a context manager, or call `close`.
+
+    Attributes
+    ----------
+    version : int
+        1 (classic) or 2 (64-bit offset).
+    numrecs : int
+        Number of records according to the header (computed from the file
+        size for a file in streaming state).
+    dimensions : dict of str to int or None
+        Dimension lengths; `None` for the unlimited dimension.
+    attributes : dict
+        Global attributes.
+    variables : dict of str to Variable
+        Variables in file order.
+    record_size : int
+        Bytes per record (all record variables).
+    """
+
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self._file: BinaryIO = self.path.open("rb")
+        try:
+            self._parse()
+        except BaseException:
+            self._file.close()
+            raise
+
+    def _parse(self) -> None:
+        name = self.path.name
+        self.size = self.path.stat().st_size
+        size = 1 << 16
+        while True:
+            self._file.seek(0)
+            data = self._file.read(size)
+            magic = data[:4]
+            if magic == CDF5:
+                raise UnsupportedVariant(
+                    f"{name} is a CDF-5 (64-bit data) netCDF file."
+                )
+            if not is_netcdf3(magic):
+                raise UnsupportedFormat(f"{name} is not a netCDF-3 file.")
+            try:
+                self._parse_header(_Parser(data, magic == OFFSET64))
+                return
+            except _Short:
+                if len(data) < size:
+                    raise CorruptFile(
+                        f"{name}: the netCDF header is truncated."
+                    ) from None
+                size *= 4
+
+    def _parse_header(self, p: _Parser) -> None:
+        p.take(4)
+        self.version = 2 if p.offset64 else 1
+        self.numrecs = p.uint()
+        dims: list[tuple[str, int]] = []
+        for _ in range(p.header_list(_DIMENSION)):
+            dims.append((p.name(), p.count()))
+        unlimited = [n for n, length in dims if length == 0]
+        if len(unlimited) > 1:
+            raise CorruptFile(f"{self.path.name}: several unlimited dimensions.")
+        self.dimensions = {n: (None if length == 0 else length) for n, length in dims}
+        self.attributes = p.attributes()
+        self.variables: dict[str, Variable] = {}
+        for _ in range(p.header_list(_VARIABLE)):
+            name = p.name()
+            ids = [p.count() for _ in range(p.count())]
+            if any(i >= len(dims) for i in ids):
+                raise CorruptFile(f"{self.path.name}: {name} has an unknown dimension.")
+            attributes = p.attributes()
+            nc_type = p.nc_type()
+            p.uint()  # vsize: recomputed below, it overflows for large variables
+            begin = p.offset()
+            names = tuple(dims[i][0] for i in ids)
+            is_record = bool(names) and self.dimensions[names[0]] is None
+            if any(self.dimensions[n] is None for n in names[1:]):
+                raise CorruptFile(
+                    f"{self.path.name}: {name} misuses the unlimited dimension."
+                )
+            shape = tuple(self.dimensions[n] or 0 for n in names)
+            self.variables[name] = Variable(
+                name, names, shape, nc_type, attributes, begin, is_record
+            )
+        self.header_size = p.pos
+        records = [v for v in self.variables.values() if v.is_record]
+        sizes = [self._slab(v) for v in records]
+        if len(records) > 1:
+            sizes = [s + _pad(s) for s in sizes]
+        self.record_size = sum(sizes)
+        self.record_begin = min((v.begin for v in records), default=self.size)
+        if self.numrecs == STREAMING:
+            self.numrecs = self.records_in_file()
+        for v in records:
+            v.shape = (self.numrecs, *v.shape[1:])
+
+    def _slab(self, variable: Variable) -> int:
+        """Bytes of one record (record variables) or of all data (others)."""
+        dims = variable.shape[1:] if variable.is_record else variable.shape
+        return int(np.prod(dims, dtype=np.int64)) * variable.dtype.itemsize
+
+    @property
+    def unlimited(self) -> str | None:
+        """Name of the unlimited dimension, or `None`."""
+        return next((n for n, v in self.dimensions.items() if v is None), None)
+
+    def records_in_file(self) -> int:
+        """Return the number of complete records the file size allows."""
+        if not self.record_size:
+            return 0
+        return max(self.size - self.record_begin, 0) // self.record_size
+
+    def read(self, name: str, numrecs: int | None = None) -> np.ndarray:
+        """Read a variable's data.
+
+        Parameters
+        ----------
+        name : str
+            Variable name.
+        numrecs : int, optional
+            Number of records to read instead of the header's (record
+            variables only); for files whose header is known to be stale.
+
+        Returns
+        -------
+        numpy.ndarray
+            Stored type in native byte order (chars as ``S1``).
+
+        Raises
+        ------
+        KeyError
+            If there is no such variable.
+        CorruptFile
+            If the file is too short for the data.
+        """
+        v = self.variables[name]
+        stored = np.dtype(TYPES[v.nc_type])
+        if not v.is_record:
+            count = int(np.prod(v.shape, dtype=np.int64))
+            end = v.begin + count * stored.itemsize
+            if end > self.size:
+                raise CorruptFile(f"{self.path.name}: {name} is truncated.")
+            self._file.seek(v.begin)
+            data = np.fromfile(self._file, stored, count).reshape(v.shape)
+            return _native(data)
+        records = self.numrecs if numrecs is None else numrecs
+        shape = (records, *v.shape[1:])
+        per_record = int(np.prod(v.shape[1:], dtype=np.int64))
+        last = v.begin + (records - 1) * self.record_size + per_record * stored.itemsize
+        if records and last > self.size:
+            raise CorruptFile(f"{self.path.name}: the records of {name} are truncated.")
+        self._file.seek(v.begin)
+        if self.record_size == per_record * stored.itemsize:  # the only record variable
+            data = np.fromfile(self._file, stored, records * per_record)
+            return _native(data.reshape(shape))
+        span = (records - 1) * self.record_size + per_record * stored.itemsize
+        raw = np.fromfile(self._file, np.uint8, span if records else 0)
+        inner = np.empty(v.shape[1:], stored).strides
+        view = np.ndarray(shape, stored, raw, 0, (self.record_size, *inner))
+        return view.astype(stored.newbyteorder("="))
+
+    def close(self) -> None:
+        """Close the file."""
+        self._file.close()
+
+    def __enter__(self) -> File:
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+
+def _native(data: np.ndarray) -> np.ndarray:
+    if data.dtype.byteorder == ">" and np.little_endian:
+        data = data.byteswap().view(data.dtype.newbyteorder("="))
+    return data
+
+
+def strings(data: np.ndarray) -> list[str]:
+    """Join a char array (``S1``, last axis the string) into strings.
+
+    Strings end at the first NUL; trailing blanks are removed.
+    """
+    data = np.asarray(data)
+    if data.ndim == 0:
+        return [data.tobytes().split(b"\0", 1)[0].decode("utf-8", "replace").rstrip()]
+    rows = data.reshape(-1, data.shape[-1]) if data.ndim > 1 else data.reshape(1, -1)
+    return [
+        row.tobytes().split(b"\0", 1)[0].decode("utf-8", "replace").rstrip()
+        for row in rows
+    ]
+
+
+# ------------------------------------------------------------------ writing
+def char_array(texts: list[str], width: int) -> np.ndarray:
+    """Encode strings as a ``(len(texts), width)`` char array (NUL padded).
+
+    Strings longer than `width` bytes (UTF-8) are truncated.
+    """
+    out = np.zeros((len(texts), width), "S1")
+    for i, text in enumerate(texts):
+        raw = text.encode("utf-8")[:width]
+        out[i, : len(raw)] = np.frombuffer(raw, "S1")
+    return out
+
+
+def nc_type_of(dtype: np.dtype) -> int:
+    """Return the netCDF type code of a numpy dtype; `ValueError` for other types."""
+    dtype = np.dtype(dtype)
+    key = (
+        "S1"
+        if dtype.kind == "S" and dtype.itemsize == 1
+        else dtype.newbyteorder("=").str[1:]
+    )
+    if key not in _TYPE_OF:
+        raise ValueError(f"netCDF-3 has no type for {dtype}.")
+    return _TYPE_OF[key]
+
+
+def _attribute_value(value: Any) -> tuple[int, bytes, int]:
+    """(nc_type, big-endian bytes, number of elements) of an attribute value."""
+    if isinstance(value, str):
+        raw = value.encode("utf-8")
+        return 2, raw, len(raw)
+    if isinstance(value, (bool, np.bool_)):
+        raise ValueError("netCDF-3 has no boolean type.")
+    if isinstance(value, int):
+        value = np.int32(value) if -(2**31) <= value < 2**31 else np.float64(value)
+    elif isinstance(value, float):
+        value = np.float64(value)
+    array = np.atleast_1d(np.asarray(value))
+    nc_type = nc_type_of(array.dtype)
+    return nc_type, array.astype(TYPES[nc_type]).tobytes(), array.size
+
+
+def _name(text: str) -> bytes:
+    raw = text.encode("utf-8")
+    return struct.pack(">I", len(raw)) + raw + bytes(_pad(len(raw)))
+
+
+def _attributes(attributes: dict[str, Any]) -> bytes:
+    items = {k: v for k, v in attributes.items() if v is not None}
+    if not items:
+        return bytes(8)
+    out = [struct.pack(">II", _ATTRIBUTE, len(items))]
+    for key, value in items.items():
+        nc_type, raw, n = _attribute_value(value)
+        out += [_name(key), struct.pack(">II", nc_type, n), raw, bytes(_pad(len(raw)))]
+    return b"".join(out)
+
+
+@dataclass
+class NewVariable:
+    """A variable to write: dimension names, data and attributes."""
+
+    dimensions: tuple[str, ...]
+    data: np.ndarray
+    attributes: dict[str, Any] = field(default_factory=dict)
+
+
+def write(
+    path: str | Path,
+    dimensions: dict[str, int | None],
+    attributes: dict[str, Any],
+    variables: dict[str, NewVariable],
+) -> Path:
+    r"""Write a netCDF classic (``CDF\x01``) file.
+
+    Parameters
+    ----------
+    path : str or Path
+        Output file (replaced if it exists).
+    dimensions : dict of str to int or None
+        Dimension lengths in order; `None` marks the (single) unlimited
+        dimension, whose length is the first axis of the record variables.
+    attributes : dict
+        Global attributes: `str` (char), `int` (int32), `float` (double), or
+        numpy scalars and 1D arrays of a netCDF-3 type. `None` is skipped.
+    variables : dict of str to NewVariable
+        Variables in file order; data must have a netCDF-3 dtype (int8,
+        ``S1``, int16, int32, float32, float64) and the dimensions' shape.
+
+    Returns
+    -------
+    Path
+        The written file.
+
+    Raises
+    ------
+    ValueError
+        For inconsistent shapes, unsupported types, or a file too large for
+        the classic format (offsets beyond 2 GiB).
+    """
+    path = Path(path)
+    names = list(dimensions)
+    unlimited = [n for n, v in dimensions.items() if v is None]
+    if len(unlimited) > 1:
+        raise ValueError("netCDF-3 allows one unlimited dimension.")
+    numrecs = None
+    prepared = []
+    for name, var in variables.items():
+        data = np.asarray(var.data)
+        nc_type = nc_type_of(data.dtype)
+        if data.ndim != len(var.dimensions):
+            raise ValueError(
+                f"{name}: data has {data.ndim} axes, dimensions {var.dimensions}."
+            )
+        is_record = bool(var.dimensions) and dimensions[var.dimensions[0]] is None
+        for axis, dim in enumerate(var.dimensions):
+            if dim not in dimensions:
+                raise ValueError(f"{name}: unknown dimension {dim!r}.")
+            length = dimensions[dim]
+            if length is None:
+                if axis:
+                    raise ValueError(
+                        f"{name}: the unlimited dimension must come first."
+                    )
+                if numrecs not in (None, data.shape[0]):
+                    raise ValueError(f"{name}: record variables differ in length.")
+                numrecs = data.shape[0]
+            elif data.shape[axis] != length:
+                raise ValueError(
+                    f"{name}: axis {axis} has {data.shape[axis]} entries, {dim} is {length}."
+                )
+        stored = data.astype(TYPES[nc_type])
+        slab = stored[0].nbytes if is_record else stored.nbytes
+        if is_record and not len(stored):
+            slab = int(np.prod(data.shape[1:], dtype=np.int64)) * stored.itemsize
+        prepared.append((name, var, nc_type, stored, is_record, slab))
+    numrecs = numrecs or 0
+    n_record_vars = sum(1 for p in prepared if p[4])
+
+    dim_list = (
+        struct.pack(">II", _DIMENSION, len(names))
+        + b"".join(_name(n) + struct.pack(">I", dimensions[n] or 0) for n in names)
+        if names
+        else bytes(8)
+    )
+    head = CLASSIC + struct.pack(">I", numrecs) + dim_list + _attributes(attributes)
+    var_headers = []
+    for name, var, nc_type, _, _, _ in prepared:
+        ids = [names.index(d) for d in var.dimensions]
+        var_headers.append(
+            _name(name)
+            + struct.pack(f">I{len(ids)}I", len(ids), *ids)
+            + _attributes(var.attributes)
+            + struct.pack(">I", nc_type)
+        )
+    # var_list tag and count (or ABSENT), then per variable vsize and begin
+    header_size = len(head) + 8 + sum(len(h) + 8 for h in var_headers)
+
+    offset = header_size
+    begins = {}
+    for name, _, _, _, is_record, slab in prepared:
+        if not is_record:
+            begins[name] = offset
+            offset += slab + _pad(slab)
+    record_size = 0
+    for name, _, _, _, is_record, slab in prepared:
+        if is_record:
+            begins[name] = offset + record_size
+            record_size += slab if n_record_vars == 1 else slab + _pad(slab)
+    if max(begins.values(), default=0) >= 1 << 31:
+        raise ValueError("Data too large for the netCDF classic format.")
+
+    with path.open("wb") as f:
+        f.write(head)
+        f.write(struct.pack(">II", _VARIABLE, len(prepared)) if prepared else bytes(8))
+        for (name, _, _, _, is_record, slab), h in zip(
+            prepared, var_headers, strict=True
+        ):
+            vsize = slab if is_record and n_record_vars == 1 else slab + _pad(slab)
+            f.write(h + struct.pack(">II", min(vsize, 0xFFFFFFFF), begins[name]))
+        for _, _, _, stored, is_record, slab in prepared:
+            if not is_record:
+                stored.tofile(f)
+                f.write(bytes(_pad(slab)))
+        if record_size and numrecs:
+            block = np.zeros((numrecs, record_size), np.uint8)
+            start = min(begins[p[0]] for p in prepared if p[4])
+            for name, _, _, stored, is_record, slab in prepared:
+                if is_record:
+                    at = begins[name] - start
+                    rows = stored.reshape(numrecs, -1).view(np.uint8)
+                    block[:, at : at + slab] = rows.reshape(numrecs, slab)
+            block.tofile(f)
+    return path

chromparser/_util.py

@@ -0,0 +1,98 @@
+"""Small helpers shared by the readers."""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, BinaryIO
+
+Source = str | Path | bytes | bytearray | BinaryIO
+
+_FILETIME_EPOCH = datetime(1601, 1, 1, tzinfo=timezone.utc)
+
+
+def read_bytes(source: Source) -> bytes:
+    """Read all bytes from a path, a bytes object or a binary file object.
+
+    Parameters
+    ----------
+    source : str, Path, bytes or file object
+        What to read.
+
+    Returns
+    -------
+    bytes
+        The content.
+
+    Raises
+    ------
+    FileNotFoundError
+        If `source` is a path that does not exist.
+    """
+    if isinstance(source, (bytes, bytearray)):
+        return bytes(source)
+    if hasattr(source, "read"):
+        return source.read()
+    path = Path(source)
+    if not path.is_file():
+        raise FileNotFoundError(f"File not found: {path}")
+    return path.read_bytes()
+
+
+def existing_file(path: str | Path) -> Path:
+    """Return `path` as a Path, or raise FileNotFoundError."""
+    path = Path(path)
+    if not path.is_file():
+        raise FileNotFoundError(f"File not found: {path}")
+    return path
+
+
+def head(path: str | Path, size: int) -> bytes:
+    """Return the first `size` bytes of a file (fewer if it is shorter)."""
+    with Path(path).open("rb") as f:
+        return f.read(size)
+
+
+def clean(metadata: dict[str, Any]) -> dict[str, Any]:
+    """Drop entries whose value is unknown (`None` or ``""``)."""
+    return {key: value for key, value in metadata.items() if value not in (None, "")}
+
+
+def filetime(ticks: int) -> datetime:
+    """Convert a Windows FILETIME (100 ns since 1601, UTC) to a datetime."""
+    return _FILETIME_EPOCH + timedelta(microseconds=ticks // 10)
+
+
+_ISO_8601 = re.compile(
+    r"^(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2})(?:\.(\d+))?(Z|[+-]\d{2}:?\d{2})?$"
+)
+
+
+def parse_iso_datetime(text: str) -> datetime | None:
+    """Parse ISO 8601 timestamps with up to 7 fractional digits (.NET style).
+
+    Parameters
+    ----------
+    text : str
+        E.g. ``"2023-09-07T15:26:11.0000000+02:00"``.
+
+    Returns
+    -------
+    datetime or None
+        Timezone-aware when `text` has an offset; `None` if it cannot be parsed.
+    """
+    match = _ISO_8601.match(text.strip())
+    if not match:
+        return None
+    base, fraction, offset = match.groups()
+    base = base.replace(" ", "T")
+    if fraction:
+        base += "." + (fraction + "000000")[:6]
+    if offset:
+        if offset == "Z":
+            offset = "+00:00"
+        elif ":" not in offset:
+            offset = offset[:3] + ":" + offset[3:]
+        base += offset
+    return datetime.fromisoformat(base)