tdmspy 0.1.0__py3-none-any.whl

tdmspy/__init__.py

@@ -0,0 +1,31 @@
+"""tdmspy: a pure Python reader for National Instruments TDMS files (MIT licensed).
+
+    import tdmspy
+
+    file = tdmspy.read("measurement.tdms")
+    channel = file["rawdata"]["Channel1"]
+    values = channel.read()
+
+The contract of the reader (what it reads, what it refuses, and which special cases of the format it
+handles) is in `doc/design.md` and `doc/special-cases.md`; the tests in `tests/` check every case.
+"""
+
+from __future__ import annotations
+
+from tdmspy.errors import TdmsError, TdmsFormatError, UnsupportedFeatureError, UnsupportedScalingError
+from tdmspy.file import Channel, Group, TdmsFile, read
+from tdmspy.timestamp import Timestamp
+
+__all__ = [
+    "Channel",
+    "Group",
+    "TdmsError",
+    "TdmsFile",
+    "TdmsFormatError",
+    "Timestamp",
+    "UnsupportedFeatureError",
+    "UnsupportedScalingError",
+    "read",
+]
+
+__implemented__ = True

tdmspy/_format.py

@@ -0,0 +1,61 @@
+"""Constants and the data type table of the TDMS format (see doc/design.md for what is supported)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+LEAD_IN_SIZE = 28
+SIGNATURE = b"TDSm"
+INDEX_SIGNATURE = b"TDSh"
+VERSIONS = (4712, 4713)
+UNKNOWN_NEXT_OFFSET = 0xFFFF_FFFF_FFFF_FFFF
+
+# table of contents flags
+TOC_METADATA = 1 << 1
+TOC_NEW_OBJECT_LIST = 1 << 2
+TOC_RAW_DATA = 1 << 3
+TOC_INTERLEAVED = 1 << 5
+TOC_BIG_ENDIAN = 1 << 6
+TOC_DAQMX = 1 << 7
+
+# raw data index values
+INDEX_NONE = 0xFFFF_FFFF
+INDEX_SAME = 0
+INDEX_DAQMX = frozenset({0x1269, 0x126A, 0x1369, 0x136A})
+
+TYPE_STRING = 0x20
+TYPE_BOOL = 0x21
+TYPE_TIMESTAMP = 0x44
+TYPE_EXTENDED = 0x0B
+TYPE_COMPLEX_SINGLE = 0x0800_000C
+TYPE_COMPLEX_DOUBLE = 0x1000_000D
+
+
+@dataclass(frozen=True)
+class DataType:
+    """A type a channel can have: the numpy type code of one element (without byte order) and its size in the file."""
+
+    code: int
+    name: str
+    numpy: str
+    size: int
+
+
+_NUMERIC = {
+    1: ("int8", "i1", 1), 2: ("int16", "i2", 2), 3: ("int32", "i4", 4), 4: ("int64", "i8", 8),
+    5: ("uint8", "u1", 1), 6: ("uint16", "u2", 2), 7: ("uint32", "u4", 4), 8: ("uint64", "u8", 8),
+    9: ("float32", "f4", 4), 10: ("float64", "f8", 8), 0x19: ("float32", "f4", 4), 0x1A: ("float64", "f8", 8),
+    TYPE_BOOL: ("bool", "u1", 1), TYPE_TIMESTAMP: ("timestamp", "", 16),
+    TYPE_COMPLEX_SINGLE: ("complex64", "c8", 8), TYPE_COMPLEX_DOUBLE: ("complex128", "c16", 16),
+}  # fmt: skip
+
+DATA_TYPES: dict[int, DataType] = {
+    code: DataType(code, name, np_code, size) for code, (name, np_code, size) in _NUMERIC.items()
+}
+DATA_TYPES[TYPE_STRING] = DataType(TYPE_STRING, "string", "", 0)
+
+# property value types: code -> (struct code, size); strings, timestamps and complex numbers are handled apart
+PROPERTY_STRUCT = {
+    1: ("b", 1), 2: ("h", 2), 3: ("i", 4), 4: ("q", 8), 5: ("B", 1), 6: ("H", 2), 7: ("I", 4), 8: ("Q", 8),
+    9: ("f", 4), 10: ("d", 8), 0x19: ("f", 4), 0x1A: ("d", 8), TYPE_BOOL: ("B", 1),
+}  # fmt: skip

tdmspy/_metadata.py

@@ -0,0 +1,205 @@
+"""Parsing the metadata section of a segment into objects, with every count and length checked against the bytes
+that are there before anything is read or allocated."""
+
+from __future__ import annotations
+
+import struct
+from dataclasses import dataclass
+from typing import Any
+
+from tdmspy._format import (
+    DATA_TYPES,
+    INDEX_DAQMX,
+    INDEX_NONE,
+    INDEX_SAME,
+    PROPERTY_STRUCT,
+    TYPE_COMPLEX_DOUBLE,
+    TYPE_COMPLEX_SINGLE,
+    TYPE_EXTENDED,
+    TYPE_STRING,
+    TYPE_TIMESTAMP,
+)
+from tdmspy.errors import TdmsFormatError, UnsupportedFeatureError
+from tdmspy.timestamp import Timestamp
+
+_MIN_OBJECT = 12  # path length, raw data index, property count
+_MIN_PROPERTY = 8  # name length, type
+
+NONE = "none"
+SAME = "same"
+
+
+@dataclass(frozen=True)
+class Index:
+    """What an object says about its raw data in a segment."""
+
+    kind: str  # NONE, SAME or "explicit"
+    type_code: int = 0
+    count: int = 0
+    total_size: int = 0  # strings: bytes of the offsets and the text of one chunk
+
+
+@dataclass
+class ObjectMeta:
+    path: tuple[str, ...]
+    index: Index
+    properties: list[tuple[str, Any]]
+
+
+def parse_path(path: str) -> tuple[str, ...]:
+    """`/` -> (), `/'g'` -> ('g',), `/'g'/'c'` -> ('g', 'c'); `''` inside a name is one apostrophe."""
+    bad = TdmsFormatError(f"invalid object path {path!r}")
+    if not path.startswith("/"):
+        raise bad
+    names: list[str] = []
+    i, n = 1, len(path)
+    while i < n:
+        if path[i] != "'":
+            raise bad
+        i += 1
+        name = []
+        while True:
+            if i >= n:
+                raise bad
+            if path[i] == "'":
+                if i + 1 < n and path[i + 1] == "'":
+                    name.append("'")
+                    i += 2
+                    continue
+                i += 1
+                break
+            name.append(path[i])
+            i += 1
+        names.append("".join(name))
+        if i < n:
+            if path[i] != "/" or i + 1 >= n:
+                raise bad
+            i += 1
+    if len(names) > 2:
+        raise bad
+    return tuple(names)
+
+
+class _Reader:
+    def __init__(self, buffer: bytes | memoryview, big_endian: bool) -> None:
+        self.buffer = buffer
+        self.pos = 0
+        self.end = len(buffer)
+        self.e = ">" if big_endian else "<"
+
+    def remaining(self) -> int:
+        return self.end - self.pos
+
+    def need(self, count: int, what: str) -> None:
+        if count > self.end - self.pos:
+            raise TdmsFormatError(f"metadata ends inside {what}")
+
+    def u32(self, what: str) -> int:
+        self.need(4, what)
+        (value,) = struct.unpack_from(self.e + "I", self.buffer, self.pos)
+        self.pos += 4
+        return value
+
+    def u64(self, what: str) -> int:
+        self.need(8, what)
+        (value,) = struct.unpack_from(self.e + "Q", self.buffer, self.pos)
+        self.pos += 8
+        return value
+
+    def string(self, what: str) -> str:
+        length = self.u32(what)
+        self.need(length, what)
+        raw = bytes(self.buffer[self.pos : self.pos + length])
+        self.pos += length
+        try:
+            return raw.decode("utf-8")
+        except UnicodeDecodeError:
+            raise TdmsFormatError(f"{what} is not valid UTF-8") from None
+
+
+def _property_value(reader: _Reader, type_code: int, name: str) -> Any:
+    if type_code in PROPERTY_STRUCT:
+        code, size = PROPERTY_STRUCT[type_code]
+        reader.need(size, f"property {name!r}")
+        (value,) = struct.unpack_from(reader.e + code, reader.buffer, reader.pos)
+        reader.pos += size
+        return bool(value) if type_code == 0x21 else value
+    if type_code == TYPE_STRING:
+        return reader.string(f"property {name!r}")
+    if type_code == TYPE_TIMESTAMP:
+        reader.need(16, f"property {name!r}")
+        if reader.e == ">":
+            seconds, fraction = struct.unpack_from(">qQ", reader.buffer, reader.pos)
+        else:
+            fraction, seconds = struct.unpack_from("<Qq", reader.buffer, reader.pos)
+        reader.pos += 16
+        return Timestamp(seconds, fraction)
+    if type_code in (TYPE_COMPLEX_SINGLE, TYPE_COMPLEX_DOUBLE):
+        code, size = ("f", 8) if type_code == TYPE_COMPLEX_SINGLE else ("d", 16)
+        reader.need(size, f"property {name!r}")
+        real, imag = struct.unpack_from(reader.e + code * 2, reader.buffer, reader.pos)
+        reader.pos += size
+        return complex(real, imag)
+    if type_code == 0:
+        return None
+    if type_code in (TYPE_EXTENDED, 0x1B):
+        raise UnsupportedFeatureError(f"property {name!r} has the extended float type, which is not supported")
+    raise TdmsFormatError(f"property {name!r} has the unknown type {type_code:#x}")
+
+
+def _index(reader: _Reader, path: str) -> Index:
+    value = reader.u32(f"the raw data index of {path!r}")
+    if value == INDEX_NONE:
+        return Index(NONE)
+    if value == INDEX_SAME:
+        return Index(SAME)
+    if value in INDEX_DAQMX:
+        raise UnsupportedFeatureError(f"{path!r} holds DAQmx raw data, which is not supported")
+    if value < 20 or value > reader.remaining() + 4:
+        raise TdmsFormatError(f"the raw data index of {path!r} has the length {value}, which does not fit the metadata")
+    following = value - 4
+    start = reader.pos
+    type_code = reader.u32("a raw data index")
+    dimension = reader.u32("a raw data index")
+    count = reader.u64("a raw data index")
+    total = 0
+    if type_code == TYPE_STRING:
+        total = reader.u64("a raw data index")
+    if reader.pos - start > following:
+        raise TdmsFormatError(f"the raw data index of {path!r} is longer than its length says")
+    reader.pos = start + following
+    if type_code == TYPE_STRING and total < 4 * count:
+        raise TdmsFormatError(
+            f"{path!r}: {count} strings need {4 * count} bytes of offsets, the index says {total} bytes in all"
+        )
+    if dimension != 1:
+        raise TdmsFormatError(f"{path!r} has an array dimension of {dimension}; only 1 is valid")
+    if type_code not in DATA_TYPES:
+        if type_code == TYPE_EXTENDED or type_code == 0x1B:
+            raise UnsupportedFeatureError(f"{path!r} has the extended float data type, which is not supported")
+        raise TdmsFormatError(f"{path!r} has the unknown data type {type_code:#x}")
+    return Index("explicit", type_code, count, total)
+
+
+def parse_metadata(buffer: bytes | memoryview, big_endian: bool) -> list[ObjectMeta]:
+    reader = _Reader(buffer, big_endian)
+    count = reader.u32("the object count")
+    if count * _MIN_OBJECT > reader.remaining():
+        raise TdmsFormatError(f"the metadata announces {count} objects but is only {reader.remaining()} bytes long")
+    objects: list[ObjectMeta] = []
+    for _ in range(count):
+        text = reader.string("an object path")
+        path = parse_path(text)
+        index = _index(reader, text)
+        if len(path) < 2 and index.kind == "explicit":
+            raise TdmsFormatError(f"{text!r} is not a channel but has raw data")
+        n_props = reader.u32(f"the properties of {text!r}")
+        if n_props * _MIN_PROPERTY > reader.remaining():
+            raise TdmsFormatError(f"{text!r} announces {n_props} properties but the metadata ends sooner")
+        properties = []
+        for _ in range(n_props):
+            name = reader.string("a property name")
+            type_code = reader.u32(f"property {name!r}")
+            properties.append((name, _property_value(reader, type_code, name)))
+        objects.append(ObjectMeta(path, index, properties))
+    return objects

tdmspy/_scaling.py

@@ -0,0 +1,69 @@
+"""NI scaling: the scales declared in the properties of a channel, its group or the file (see doc/design.md)."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+import numpy as np
+
+from tdmspy.errors import UnsupportedScalingError
+
+RAW_DATA_INPUT_SOURCE = 0xFFFF_FFFF
+
+
+def scale_count(properties: Mapping[str, Any]) -> int | None:
+    """How many scales these properties declare, `None` if none or the data is already scaled."""
+    count = properties.get("NI_Number_Of_Scales")
+    if isinstance(count, bool) or not isinstance(count, int) or count <= 0:
+        return None
+    if properties.get("NI_Scaling_Status", "unscaled") == "scaled":
+        return None
+    return count
+
+
+def _number(properties: Mapping[str, Any], key: str) -> float:
+    value = properties.get(key)
+    if isinstance(value, bool) or not isinstance(value, int | float):
+        raise UnsupportedScalingError(f"the scaling property {key!r} is missing or not a number")
+    return float(value)
+
+
+def apply_scaling(raw: np.ndarray, *levels: Mapping[str, Any]) -> np.ndarray:
+    """`raw` scaled by the first of `levels` (channel, group, file properties) that declares scales, else `raw` itself."""
+    for properties in levels:
+        count = scale_count(properties)
+        if count is not None:
+            return _scale(raw, properties, count)
+    return raw
+
+
+def _scale(raw: np.ndarray, properties: Mapping[str, Any], count: int) -> np.ndarray:
+    def compute(index: int, depth: int) -> np.ndarray:
+        if index == RAW_DATA_INPUT_SOURCE:
+            return raw.astype(np.float64)
+        if not 0 <= index < count or depth > count:
+            raise UnsupportedScalingError(
+                f"NI scale {index} does not exist or the scales refer to each other in a circle"
+            )
+        prefix = f"NI_Scale[{index}]_"
+        kind = properties.get(prefix + "Scale_Type")
+        if kind not in ("Linear", "Polynomial"):
+            what = "a DAQmx scaler" if kind is None else f"the scale type {kind!r}"
+            raise UnsupportedScalingError(f"{what} is not supported (only Linear and Polynomial scales are)")
+        source = properties.get(f"{prefix}{kind}_Input_Source", RAW_DATA_INPUT_SOURCE)
+        values = compute(int(source), depth + 1)
+        if kind == "Linear":
+            return values * _number(properties, prefix + "Linear_Slope") + _number(
+                properties, prefix + "Linear_Y_Intercept"
+            )
+        size = properties.get(prefix + "Polynomial_Coefficients_Size", 4)
+        if isinstance(size, bool) or not isinstance(size, int) or not 0 <= size <= 1024:
+            raise UnsupportedScalingError(f"the polynomial of scale {index} has an invalid number of coefficients")
+        coefficients = [_number(properties, f"{prefix}Polynomial_Coefficients[{k}]") for k in range(size)]
+        result = np.zeros(len(values))
+        for coefficient in reversed(coefficients):  # Horner's rule
+            result = result * values + coefficient
+        return result
+
+    return compute(count - 1, 0)

tdmspy/_scan.py

@@ -0,0 +1,269 @@
+"""Scanning a file: every segment's lead-in and metadata become, per channel, a list of data blocks.
+
+The result of a scan holds no channel data. A *block* says where a run of a channel's values lies in the file:
+`[offset, per, repeat, stride, big_endian]` is `repeat` units of `per` consecutive values, the units `stride`
+bytes apart, the first starting at `offset`. A segment of one chunk is one unit; several chunks, or interleaved
+data, are many units; the identical segments of an acquisition are merged into one block.
+"""
+
+from __future__ import annotations
+
+import mmap
+import struct
+from dataclasses import dataclass, field
+from typing import Any
+
+from tdmspy._format import (
+    DATA_TYPES,
+    INDEX_SIGNATURE,
+    LEAD_IN_SIZE,
+    SIGNATURE,
+    TOC_BIG_ENDIAN,
+    TOC_DAQMX,
+    TOC_INTERLEAVED,
+    TOC_METADATA,
+    TOC_NEW_OBJECT_LIST,
+    TOC_RAW_DATA,
+    TYPE_STRING,
+    UNKNOWN_NEXT_OFFSET,
+    VERSIONS,
+)
+from tdmspy._metadata import NONE, SAME, Index, ObjectMeta, parse_metadata
+from tdmspy.errors import TdmsFormatError, UnsupportedFeatureError
+
+
+@dataclass(eq=False)
+class ChannelIndex:
+    """Everything the scan learned about one channel."""
+
+    name: str
+    properties: dict[str, Any] = field(default_factory=dict)
+    type_code: int | None = None
+    last: Index | None = None  # the last explicit raw data index
+    current: Index | None = None  # the index in force now (None: no data in the current segments)
+    blocks: list[list[int]] = field(default_factory=list)
+    string_chunks: list[tuple[int, int, int, bool]] = field(default_factory=list)  # offset, count, bytes, big endian
+    length: int = 0
+
+
+@dataclass(eq=False)
+class GroupIndex:
+    name: str
+    properties: dict[str, Any] = field(default_factory=dict)
+    channels: dict[str, ChannelIndex] = field(default_factory=dict)
+
+
+@dataclass
+class Scan:
+    properties: dict[str, Any] = field(default_factory=dict)
+    groups: dict[str, GroupIndex] = field(default_factory=dict)
+    truncated: bool = False
+
+
+def _item_size(channel: ChannelIndex) -> int:
+    assert channel.type_code is not None
+    return DATA_TYPES[channel.type_code].size
+
+
+class _State:
+    def __init__(self) -> None:
+        self.scan = Scan()
+        self.active: list[ChannelIndex] = []
+        self.layout: tuple[list[ChannelIndex], list[int], int] | None = None  # members, sizes, chunk size
+
+    def group(self, name: str) -> GroupIndex:
+        group = self.scan.groups.get(name)
+        if group is None:
+            group = self.scan.groups[name] = GroupIndex(name)
+        return group
+
+    def apply(self, objects: list[ObjectMeta], new_list: bool) -> None:
+        listed: list[ChannelIndex] = []
+        for obj in objects:
+            if len(obj.path) == 0:
+                self.scan.properties.update(obj.properties)
+            elif len(obj.path) == 1:
+                self.group(obj.path[0]).properties.update(obj.properties)
+            else:
+                group = self.group(obj.path[0])
+                channel = group.channels.get(obj.path[1])
+                if channel is None:
+                    channel = group.channels[obj.path[1]] = ChannelIndex(obj.path[1])
+                channel.properties.update(obj.properties)
+                self._index(channel, obj.index, obj.path)
+                if channel not in listed:
+                    listed.append(channel)
+        if new_list:
+            self.active = listed
+        else:
+            known = {id(c) for c in self.active}
+            self.active += [c for c in listed if id(c) not in known]
+        self.layout = None
+
+    @staticmethod
+    def _index(channel: ChannelIndex, index: Index, path: tuple[str, ...]) -> None:
+        name = "/".join(path)
+        if index.kind == NONE:
+            channel.current = None
+        elif index.kind == SAME:
+            if channel.last is None:
+                raise TdmsFormatError(f"{name!r} has the raw data index 'same as before' but never had one")
+            channel.current = channel.last
+        else:
+            if channel.type_code is not None and channel.type_code != index.type_code:
+                raise TdmsFormatError(
+                    f"the data type of {name!r} changes from {channel.type_code:#x} to {index.type_code:#x}"
+                )
+            channel.type_code = index.type_code
+            channel.last = channel.current = index
+
+    def members(self) -> tuple[list[ChannelIndex], list[int], int]:
+        if self.layout is None:
+            members = [c for c in self.active if c.current is not None]
+            sizes = []
+            for channel in members:
+                index = channel.current
+                assert index is not None
+                sizes.append(index.total_size if index.type_code == TYPE_STRING else index.count * _item_size(channel))
+            self.layout = (members, sizes, sum(sizes))
+        return self.layout
+
+
+def _add_block(channel: ChannelIndex, offset: int, per: int, repeat: int, stride: int, big: bool) -> None:
+    """Append a block, merging it into the previous one when it continues it at a regular distance."""
+    blocks = channel.blocks
+    if blocks:
+        last = blocks[-1]
+        if repeat == 1 and last[1] == per and last[4] == int(big):
+            if last[2] == 1:
+                last[2], last[3] = 2, offset - last[0]
+                channel.length += per
+                return
+            if offset - (last[0] + (last[2] - 1) * last[3]) == last[3]:
+                last[2] += 1
+                channel.length += per
+                return
+    blocks.append([offset, per, repeat, stride, int(big)])
+    channel.length += per * repeat
+
+
+def _segment_layout(
+    state: _State, raw_start: int, raw_len: int, interleaved: bool, big: bool, partial_allowed: bool
+) -> bool:
+    """Add the blocks of one segment's raw data; returns whether the data was cut inside a chunk."""
+    members, sizes, chunk = state.members()
+    if chunk == 0:
+        if raw_len > 0:
+            raise TdmsFormatError("a segment has raw data but no channel accounts for it")
+        return False
+    whole, remainder = divmod(raw_len, chunk)
+    if remainder and not partial_allowed or whole == 0 and not partial_allowed:
+        raise TdmsFormatError(
+            f"the raw data of a segment is {raw_len} bytes, which is not a whole number of chunks of {chunk} bytes"
+        )
+    if interleaved:
+        counts = {c.current.count for c in members if c.current is not None}
+        if len(counts) > 1 or any(c.type_code == TYPE_STRING for c in members):
+            raise TdmsFormatError("interleaved channels must all have the same count and cannot be strings")
+        count = counts.pop() if counts else 0
+        row = sum(_item_size(c) for c in members)
+        rows = whole * count + remainder // row
+        offset = raw_start
+        for channel in members:
+            if rows:
+                _add_block(channel, offset, 1, rows, row, big)
+            offset += _item_size(channel)
+        return remainder > 0
+    offset = 0
+    left = remainder
+    for channel, size in zip(members, sizes, strict=True):
+        index = channel.current
+        assert index is not None
+        if channel.type_code == TYPE_STRING:
+            for c in range(whole):
+                channel.string_chunks.append((raw_start + c * chunk + offset, index.count, size, big))
+                channel.length += index.count
+        else:
+            if whole and index.count:
+                _add_block(channel, raw_start + offset, index.count, whole, chunk, big)
+            if left > 0:
+                complete = min(left, size) // _item_size(channel)
+                if complete:
+                    _add_block(channel, raw_start + whole * chunk + offset, complete, 1, 0, big)
+        left -= size
+        offset += size
+    return remainder > 0
+
+
+def scan_file(path: str, *, strict: bool) -> Scan:
+    with open(path, "rb") as handle:
+        handle.seek(0, 2)
+        size = handle.tell()
+        if size < LEAD_IN_SIZE:
+            raise TdmsFormatError(f"{path}: {size} bytes are too few for a TDMS file (a segment starts with 28 bytes)")
+        with mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) as data:
+            return _scan(data, size, strict)
+
+
+def _cut(strict: bool, message: str) -> None:
+    if strict:
+        raise TdmsFormatError(message)
+
+
+def _scan(data: mmap.mmap, size: int, strict: bool) -> Scan:
+    state = _State()
+    scan = state.scan
+    pos = 0
+    segments = 0
+    cache_key: tuple | None = None
+    while pos < size:
+        if size - pos < LEAD_IN_SIZE:
+            _cut(strict, f"the file ends inside the lead-in of a segment (byte {pos})")
+            scan.truncated = True
+            break
+        tag = data[pos : pos + 4]
+        if tag != SIGNATURE:
+            if tag == INDEX_SIGNATURE:
+                raise TdmsFormatError("this is a TDMS index file (.tdms_index): it holds no data, open the .tdms file")
+            raise TdmsFormatError(f"no TDMS segment at byte {pos} (found {bytes(tag)!r})")
+        (toc,) = struct.unpack_from("<I", data, pos + 4)
+        big = bool(toc & TOC_BIG_ENDIAN)
+        version, next_offset, raw_offset = struct.unpack_from((">" if big else "<") + "IQQ", data, pos + 8)
+        if version not in VERSIONS:
+            raise TdmsFormatError(f"unknown TDMS version {version} in the segment at byte {pos} (known: {VERSIONS})")
+        unknown_end = next_offset == UNKNOWN_NEXT_OFFSET
+        end = size if unknown_end else pos + LEAD_IN_SIZE + next_offset
+        cut = end > size
+        if cut:
+            end = size
+        raw_start = pos + LEAD_IN_SIZE + raw_offset
+        if raw_start > end:
+            if cut:
+                _cut(strict, f"the file ends inside the metadata of the segment at byte {pos}")
+                scan.truncated = True
+                break
+            raise TdmsFormatError(f"the raw data offset of the segment at byte {pos} points beyond the segment")
+
+        if toc & TOC_METADATA:
+            metadata = bytes(data[pos + LEAD_IN_SIZE : raw_start])
+            key = (metadata, toc & (TOC_NEW_OBJECT_LIST | TOC_BIG_ENDIAN))
+            if key != cache_key:  # identical metadata twice in a row changes nothing the second time
+                objects = parse_metadata(metadata, big)
+                state.apply(objects, bool(toc & TOC_NEW_OBJECT_LIST))
+                cache_key = key
+        raw_len = end - raw_start
+        if toc & TOC_DAQMX and raw_len:
+            raise UnsupportedFeatureError("DAQmx raw data is not supported")
+        partial = False
+        if raw_len or toc & TOC_RAW_DATA:
+            partial = _segment_layout(
+                state, raw_start, raw_len, bool(toc & TOC_INTERLEAVED), big, partial_allowed=cut or unknown_end
+            )
+        if cut or partial:
+            _cut(strict, f"the file ends inside the segment at byte {pos}")
+            scan.truncated = True
+        segments += 1
+        pos = end
+    if segments == 0:
+        raise TdmsFormatError("the file has no complete segment")
+    return scan

tdmspy/errors.py

@@ -0,0 +1,17 @@
+"""The exceptions of tdmspy. Everything the package raises on purpose is a `TdmsError`."""
+
+
+class TdmsError(Exception):
+    """Base class of every error raised for a file's content."""
+
+
+class TdmsFormatError(TdmsError, ValueError):
+    """The file is not valid TDMS (wrong signature, corrupt or inconsistent segments, ...)."""
+
+
+class UnsupportedFeatureError(TdmsError, NotImplementedError):
+    """The file is valid TDMS but uses something this package does not read (DAQmx raw data, ...)."""
+
+
+class UnsupportedScalingError(UnsupportedFeatureError):
+    """A channel's NI scaling uses a scale type that is not implemented (`read(scaled=False)` still works)."""

tdmspy/file.py

@@ -0,0 +1,236 @@
+"""The objects `tdmspy.read` returns: `TdmsFile`, `Group` and `Channel` (see doc/design.md)."""
+
+from __future__ import annotations
+
+import mmap
+import os
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from tdmspy._format import DATA_TYPES, TYPE_BOOL, TYPE_STRING, TYPE_TIMESTAMP
+from tdmspy._scaling import apply_scaling
+from tdmspy._scan import ChannelIndex, GroupIndex, Scan, scan_file
+from tdmspy.errors import TdmsFormatError
+from tdmspy.timestamp import UNIX_EPOCH_IN_TDMS_SECONDS
+
+_LE_STAMP = np.dtype([("fraction", "<u8"), ("seconds", "<i8")])
+_BE_STAMP = np.dtype([("seconds", ">i8"), ("fraction", ">u8")])
+
+
+def _microseconds(seconds: np.ndarray, fraction: np.ndarray) -> np.ndarray:
+    """`datetime64[us]` from TDMS seconds and 2**-64 fractions: the fraction is truncated, exactly (no floats).
+
+    floor(fraction * 10**6 / 2**64) with the fraction split in two 32 bit halves, so nothing overflows 64 bits.
+    """
+    thirty_two = np.uint64(32)
+    high, low = fraction >> thirty_two, fraction & np.uint64(0xFFFF_FFFF)
+    a, b = high * np.uint64(1_000_000), low * np.uint64(1_000_000)
+    micro = ((a + (b >> thirty_two)) >> thirty_two).astype(np.int64)
+    ticks = (seconds.astype(np.int64) - UNIX_EPOCH_IN_TDMS_SECONDS) * 1_000_000 + micro
+    return ticks.astype("datetime64[us]")
+
+
+def _decode(raw: bytes) -> str:
+    try:
+        return raw.decode("utf-8")
+    except UnicodeDecodeError:
+        raise TdmsFormatError("a string value is not valid UTF-8") from None
+
+
+class Channel:
+    """A channel: its properties, data type, length, and `read` for its values."""
+
+    def __init__(self, index: ChannelIndex, group: Group, file: TdmsFile) -> None:
+        self.name = index.name
+        self.properties: dict[str, Any] = index.properties
+        self._group = group
+        self._file = file
+        self._type_code = index.type_code
+        self._length = index.length
+        self._strings = index.string_chunks
+        self._blocks = np.array(index.blocks, dtype=np.int64).reshape(-1, 5)
+        self._starts = np.concatenate(([0], np.cumsum(self._blocks[:, 1] * self._blocks[:, 2])))
+        self._dtype = None if index.type_code is None else self._native_dtype(index.type_code)
+
+    @staticmethod
+    def _native_dtype(type_code: int) -> np.dtype:
+        if type_code == TYPE_STRING:
+            return np.dtype(object)
+        if type_code == TYPE_BOOL:
+            return np.dtype(bool)
+        if type_code == TYPE_TIMESTAMP:
+            return np.dtype("datetime64[us]")
+        return np.dtype("=" + DATA_TYPES[type_code].numpy)
+
+    @property
+    def dtype(self) -> np.dtype | None:
+        """The numpy dtype of the values (native byte order); `None` for a channel that never has a data type."""
+        return self._dtype
+
+    def __len__(self) -> int:
+        return self._length
+
+    def __repr__(self) -> str:
+        return f"<Channel {self.name!r}: {self._dtype}, {self._length} values>"
+
+    def read(self, start: int = 0, stop: int | None = None, *, scaled: bool = True) -> np.ndarray:
+        """The values `start` up to `stop` (all by default) as a new array; NI scaling is applied unless `scaled=False`."""
+        if start < 0 or (stop is not None and stop < 0):
+            raise ValueError("start and stop must not be negative")
+        stop = self._length if stop is None else min(stop, self._length)
+        start = min(start, self._length)
+        stop = max(stop, start)
+        if self._dtype is None:
+            return np.empty(0)
+        raw = self._read_strings(start, stop) if self._type_code == TYPE_STRING else self._read_numbers(start, stop)
+        if scaled and raw.dtype.kind in "iuf":
+            return apply_scaling(raw, self.properties, self._group.properties, self._file.properties)
+        return raw
+
+    # -- numbers ---------------------------------------------------------------------------------
+
+    def _read_numbers(self, start: int, stop: int) -> np.ndarray:
+        assert self._dtype is not None
+        out = np.empty(stop - start, dtype=self._dtype)
+        if stop == start:
+            return out
+        first = int(np.searchsorted(self._starts, start, side="right")) - 1
+        last = int(np.searchsorted(self._starts, stop, side="left"))
+        with open(self._file.path, "rb") as handle, mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) as data:
+            for i in range(first, last):
+                base = int(self._starts[i])
+                lo, hi = max(start, base), min(stop, int(self._starts[i + 1]))
+                if lo < hi:
+                    offset, per, repeat, stride, big = (int(v) for v in self._blocks[i])
+                    self._read_block(
+                        data,
+                        (offset, per, repeat, stride, bool(big)),
+                        lo - base,
+                        hi - base,
+                        out[lo - start : hi - start],
+                    )
+        return out
+
+    def _read_block(
+        self, data: mmap.mmap, block: tuple[int, int, int, int, bool], v0: int, v1: int, out: np.ndarray
+    ) -> None:
+        """Values `v0` to `v1` of one block into `out`."""
+        assert self._type_code is not None
+        offset, per, repeat, stride, big = block
+        info = DATA_TYPES[self._type_code]
+        if self._type_code == TYPE_TIMESTAMP:
+            dtype, size = (_BE_STAMP if big else _LE_STAMP), 16
+        else:
+            dtype, size = np.dtype(("<>"[big]) + info.numpy), info.size
+        if offset + (repeat - 1) * stride + per * size > len(data):
+            raise TdmsFormatError("the file is shorter than when it was opened")
+
+        def copy(unit: int, count: int, within: int, into: np.ndarray, units: int = 1) -> None:
+            view = np.ndarray(
+                (units, count),
+                dtype=dtype,
+                buffer=data,
+                offset=offset + unit * stride + within * size,
+                strides=(stride, size),
+            )
+            self._convert(view, into.reshape(units, count))
+
+        wanted = v1 - v0
+        position = 0
+        unit, within = divmod(v0, per)
+        if within:  # the tail of a first unit that is only partly wanted
+            count = min(per - within, wanted)
+            copy(unit, count, within, out[:count])
+            position, unit = count, unit + 1
+        full = (v1 // per) - unit
+        if full > 0 and position < wanted:
+            copy(unit, per, 0, out[position : position + full * per], units=full)
+            position, unit = position + full * per, unit + full
+        if position < wanted:  # the head of a last unit that is only partly wanted
+            copy(unit, wanted - position, 0, out[position:])
+
+    def _convert(self, view: np.ndarray, into: np.ndarray) -> None:
+        if self._type_code == TYPE_TIMESTAMP:
+            into[...] = _microseconds(view["seconds"], view["fraction"].astype(np.uint64))
+        elif self._type_code == TYPE_BOOL:
+            into[...] = view != 0
+        else:
+            into[...] = view
+
+    # -- strings ---------------------------------------------------------------------------------
+
+    def _read_strings(self, start: int, stop: int) -> np.ndarray:
+        out = np.empty(stop - start, dtype=object)
+        position = 0
+        with open(self._file.path, "rb") as handle:
+            for offset, count, total, big in self._strings:
+                lo, hi = max(start, position), min(stop, position + count)
+                if lo < hi:
+                    handle.seek(offset)
+                    chunk = handle.read(total)
+                    if len(chunk) < total or total < 4 * count:
+                        raise TdmsFormatError("a string channel's data is shorter than its index says")
+                    ends = np.frombuffer(chunk, dtype=("<>"[big]) + "u4", count=count)
+                    text = chunk[4 * count :]
+                    for i in range(lo - position, hi - position):
+                        begin, end = (int(ends[i - 1]) if i else 0), int(ends[i])
+                        if not begin <= end <= len(text):
+                            raise TdmsFormatError("the offsets of a string channel are inconsistent")
+                        out[lo - start + i - (lo - position)] = _decode(text[begin:end])
+                position += count
+                if position >= stop:
+                    break
+        return out
+
+
+class Group:
+    """A group: its properties and its channels, in file order."""
+
+    def __init__(self, index: GroupIndex, file: TdmsFile) -> None:
+        self.name = index.name
+        self.properties: dict[str, Any] = index.properties
+        self.channels: list[Channel] = [Channel(c, self, file) for c in index.channels.values()]
+        self._by_name = {c.name: c for c in self.channels}
+
+    def __getitem__(self, name: str) -> Channel:
+        return self._by_name[name]
+
+    def __contains__(self, name: object) -> bool:
+        return name in self._by_name
+
+    def __iter__(self) -> Iterator[Channel]:
+        return iter(self.channels)
+
+    def __repr__(self) -> str:
+        return f"<Group {self.name!r}: {len(self.channels)} channels>"
+
+
+class TdmsFile:
+    """A TDMS file: its properties and groups. Channel data is read when asked for."""
+
+    def __init__(self, path: str | os.PathLike[str], scan: Scan) -> None:
+        self.path = Path(path)
+        self.properties: dict[str, Any] = scan.properties
+        self.truncated = scan.truncated
+        self.groups: list[Group] = [Group(g, self) for g in scan.groups.values()]
+        self._by_name = {g.name: g for g in self.groups}
+
+    def __getitem__(self, name: str) -> Group:
+        return self._by_name[name]
+
+    def __contains__(self, name: object) -> bool:
+        return name in self._by_name
+
+    def __iter__(self) -> Iterator[Group]:
+        return iter(self.groups)
+
+    def __repr__(self) -> str:
+        return f"<TdmsFile {self.path.name!r}: {len(self.groups)} groups>"
+
+
+def read(path: str | os.PathLike[str], *, strict: bool = False) -> TdmsFile:
+    """Open the TDMS file at `path`: its structure is read, the channel data is not (see doc/design.md)."""
+    return TdmsFile(path, scan_file(os.fspath(path), strict=strict))

tdmspy/timestamp.py

@@ -0,0 +1,57 @@
+"""The TDMS timestamp: seconds since 1904-01-01 00:00:00 UTC plus a fraction of a second.
+
+A property or channel value of type "timestamp" keeps both parts exactly, as the file stores them.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+
+FRACTIONS_PER_SECOND = 1 << 64
+"""The fraction counts units of 2**-64 seconds."""
+
+UNIX_EPOCH_IN_TDMS_SECONDS = 2_082_844_800
+"""Seconds between 1904-01-01 and 1970-01-01."""
+
+_UNITS_PER_SECOND = {"s": 1, "ms": 1_000, "us": 1_000_000, "ns": 1_000_000_000}
+
+
+@dataclass(frozen=True, order=True)
+class Timestamp:
+    """`seconds` since 1904-01-01 UTC (may be negative) and `fraction` of a second in units of 2**-64 s."""
+
+    seconds: int
+    fraction: int = 0
+
+    def __post_init__(self) -> None:
+        if not 0 <= self.fraction < FRACTIONS_PER_SECOND:
+            raise ValueError(f"fraction must be in [0, 2**64), got {self.fraction}")
+
+    def to_datetime64(self, unit: str = "us") -> np.datetime64:
+        """UTC time as a `numpy.datetime64` of `unit` (`s`, `ms`, `us` or `ns`), truncated (never rounded up).
+
+        The fraction is converted with integer arithmetic, so the result is exact.
+        """
+        try:
+            per_second = _UNITS_PER_SECOND[unit]
+        except KeyError:
+            raise ValueError(f"unit must be one of {sorted(_UNITS_PER_SECOND)}, got {unit!r}") from None
+        ticks = (
+            self.seconds - UNIX_EPOCH_IN_TDMS_SECONDS
+        ) * per_second + self.fraction * per_second // FRACTIONS_PER_SECOND
+        return np.datetime64(ticks, unit)
+
+    @classmethod
+    def from_datetime64(cls, value: np.datetime64) -> Timestamp:
+        """The timestamp of a `numpy.datetime64` (any unit up to `ns`), exact for the value given."""
+        unit = np.datetime_data(value.dtype)[0]
+        if unit not in _UNITS_PER_SECOND:
+            raise ValueError(f"unsupported datetime64 unit {unit!r}")
+        per_second = _UNITS_PER_SECOND[unit]
+        ticks = int(value.astype(f"datetime64[{unit}]").astype(np.int64))
+        seconds, remainder = divmod(ticks, per_second)
+        # round up so that to_datetime64 of the result gives the same value back
+        fraction = -(-remainder * FRACTIONS_PER_SECOND // per_second)
+        return cls(seconds + UNIX_EPOCH_IN_TDMS_SECONDS, fraction)

tdmspy-0.1.0.dist-info/METADATA

@@ -0,0 +1,76 @@
+Metadata-Version: 2.5
+Name: tdmspy
+Version: 0.1.0
+Summary: Pure Python reader for National Instruments TDMS files, MIT licensed
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.13
+Requires-Dist: numpy>=2.5.1
+Description-Content-Type: text/markdown
+
+# tdmspy
+
+A pure Python reader for National Instruments **TDMS** files, MIT licensed. Its only runtime dependency is numpy.
+
+It is written from the format specification and tested against real files and
+synthetic test cases, one for every special case of the format.
+
+## Install
+
+```sh
+uv venv --allow-existing
+uv sync --all-extras
+```
+
+## Usage
+
+```python
+import tdmspy
+
+file = tdmspy.read("measurement.tdms")  # indexes the file; no channel data is read yet
+for group in file.groups:  # in file order, not sorted
+    for channel in group.channels:
+        print(group.name, channel.name, channel.dtype, len(channel), channel.properties)
+
+values = file["rawdata"]["Channel1"].read()  # numpy array, native byte order, NI scaling applied
+raw = file["rawdata"]["Channel1"].read(scaled=False)
+part = file["rawdata"]["Channel1"].read(start=1000, stop=2000)
+```
+
+The details (what is returned for each data type, what is refused, how truncated files are treated) are in
+[`doc/design.md`](doc/design.md); every special case of the format the reader has to handle is listed with the files
+that exercise it in [`doc/special-cases.md`](doc/special-cases.md).
+
+## Tests
+
+```sh
+just test        # lint, then everything except the slow tests
+just test-all    # including large files and long fuzz runs
+just testcases      # rewrite the stored synthetic files in tests/data/synthetic
+```
+
+The tests are the specification of the reader:
+
+- `tests/catalogue.py` lists every special case of the format. Add a case there first.
+- `tests/builder.py` is a byte-level TDMS writer for the tests (interleaved, big-endian, several chunks, truncated,
+  corrupt, every property type, ...). It does not use tdmspy.
+- `tests/testcases.py` has one small file per special case, each with what it must read as (`Truth`). The files are
+  stored in `tests/data/synthetic`; the long ones are built when a test needs them.
+- `tests/data/real` are real files, with their identifying strings replaced by
+  placeholders; `tests/test_real_files.py` checks them and `tests/test_privacy.py` keeps identifiers out of the test data.
+- `tests/test_reader_*.py` are the tests of `tdmspy.read`: all test cases, the API, strict mode and scaling.
+- `tests/test_fuzz.py` and `tests/test_robust.py` damage files and read large ones.
+
+`just catalogue` regenerates `doc/special-cases.md`.
+
+## Inspired by
+
+[npTDMS](https://github.com/adamreeve/npTDMS), [pyTDMS](https://github.com/petebachant/pyTDMS) and
+[ICOschema](https://github.com/ift-tuwien/ICOschema), which tdmspy was created to aid: ICOschema reads TDMS files
+through it. No code of any of these projects is used here.
+
+TDMS, LabVIEW and NI are trademarks of National Instruments. tdmspy is not affiliated with or endorsed by NI.
+
+## Licence
+
+MIT, see [LICENSE](LICENSE).

tdmspy-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+tdmspy/__init__.py,sha256=XUuRR04aC2K5xjy0uVt7kqu3DOpDSMDILT3t1E4CY7Y,870
+tdmspy/_format.py,sha256=5Kfb6xi1Z-dKuv5LGtC9z1ur0fQ61eo-l3LIlQOuvlc,2010
+tdmspy/_metadata.py,sha256=oTAJ7qekHPC7i2M-EKLaaq71d1KboiRcUUgRs8Q0hWI,7543
+tdmspy/_scaling.py,sha256=KjAkhAGkzyR8y1iTo2IQ_BgS2hwfbEgxvHcQHCyvgZQ,3077
+tdmspy/_scan.py,sha256=zaDr5epjrjsHw5srrHpmOsVq8Yp7gq5bZzeVWvRZZ7I,11024
+tdmspy/errors.py,sha256=520q0a5VfloULbgqUG0-nFdwzhcZTPmrXXEIeXv_kco,665
+tdmspy/file.py,sha256=IvcTQk1iqWGXlZziyINByQ6IQUPjipLD7WbSCF6-ia8,10220
+tdmspy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tdmspy/timestamp.py,sha256=VSYZykzYy173MKDttY6XkZeRPJFeXj5-G7O53wgYQwQ,2317
+tdmspy-0.1.0.dist-info/METADATA,sha256=1dLjfO5LymjFAOo4tyPmaHLJP-GbIg1vAWtQT1dNPZY,3048
+tdmspy-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+tdmspy-0.1.0.dist-info/licenses/LICENSE,sha256=pGTOdcwdvrXSTL8WKTZQxYlYVikuc6srBYve_adrc0s,1147
+tdmspy-0.1.0.dist-info/RECORD,,

tdmspy-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

tdmspy-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Moritz Breurather, Institute of Production Engineering and Photonic Technologies TU Wien  
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.