npTDMS 1.11.0__py3-none-any.whl

nptdms/__init__.py

@@ -0,0 +1,11 @@
+"""Module for reading binary TDMS files produced by LabView"""
+
+from __future__ import absolute_import
+
+
+# Make version number available
+from .version import __version_info__, __version__
+
+# Export public objects
+from .tdms import TdmsFile, TdmsGroup, TdmsChannel, DataChunk, GroupDataChunk, ChannelDataChunk
+from .writer import TdmsWriter, RootObject, GroupObject, ChannelObject

nptdms/base_segment.py

@@ -0,0 +1,179 @@
+from io import UnsupportedOperation
+import numpy as np
+
+from nptdms.log import log_manager
+
+
+log = log_manager.get_logger(__name__)
+
+
+class BaseSegmentObject(object):
+    """ Abstract base class for an object in a TDMS segment
+    """
+
+    __slots__ = [
+        'path', 'number_values', 'data_size',
+        'has_data', 'data_type']
+
+    def __init__(self, path):
+        self.path = path
+        self.number_values = 0
+        self.data_size = 0
+        self.has_data = False
+        self.data_type = None
+
+    def read_raw_data_index(self, file, raw_data_index_header, endianness):
+        """ Read the raw data index for a single object in a segment
+        """
+        raise NotImplementedError("Segment metadata reading must be implemented in base classes")
+
+    @property
+    def scaler_data_types(self):
+        return None
+
+    def __repr__(self):
+        return "%s(%s)" % (self.__class__.__name__, self.path)
+
+
+class BaseDataReader(object):
+    """ Abstract base class for reading data in a segment
+    """
+
+    def __init__(self, num_chunks, final_chunk_lengths_override, endianness):
+        self.num_chunks = num_chunks
+        self.final_chunk_lengths_override = final_chunk_lengths_override
+        self.endianness = endianness
+
+    def read_data_chunks(self, file, data_objects, num_chunks):
+        """ Read multiple data chunks at once
+            In the base case we read each chunk individually but subclasses can override this
+        """
+        for chunk in range(num_chunks):
+            yield self._read_data_chunk(file, data_objects, chunk)
+
+    def _read_data_chunk(self, file, data_objects, chunk_index):
+        """ Read data from a chunk for all channels
+        """
+        raise NotImplementedError("Data chunk reading must be implemented in base classes")
+
+    def read_channel_data_chunks(self, file, data_objects, channel_path, chunk_offset, stop_chunk):
+        """ Read multiple data chunks for a single channel at once
+            In the base case we read each chunk individually but subclasses can override this
+        """
+        for chunk_index in range(chunk_offset, stop_chunk):
+            yield self._read_channel_data_chunk(file, data_objects, chunk_index, channel_path)
+
+    def _read_channel_data_chunk(self, file, data_objects, chunk_index, channel_path):
+        """ Read data from a chunk for a single channel
+        """
+        # In the base case we can read data for all channels
+        # and then select only the requested channel.
+        # Derived classes can implement more optimised reading.
+        data_chunk = self._read_data_chunk(file, data_objects, chunk_index)
+        return data_chunk_to_channel_chunk(data_chunk, channel_path)
+
+
+class RawDataChunk(object):
+    """Data read from a single chunk in a TDMS segment
+
+    :ivar channel_data: A dictionary of channel data chunks.
+        Keys are object paths and values are RawChannelDataChunk instances.
+    """
+
+    def __init__(self, channel_data):
+        self.channel_data = channel_data
+
+    @staticmethod
+    def empty():
+        return RawDataChunk({})
+
+    @staticmethod
+    def channel_data(data):
+        channel_chunks = {
+            path: RawChannelDataChunk.channel_data(d)
+            for (path, d) in data.items()
+        }
+        return RawDataChunk(channel_chunks)
+
+    @staticmethod
+    def scaler_data(data):
+        channel_chunks = {
+            path: RawChannelDataChunk.scaler_data(d)
+            for (path, d) in data.items()
+        }
+        return RawDataChunk(channel_chunks)
+
+
+class RawChannelDataChunk(object):
+    """Data read for a single channel from a single chunk in a TDMS segment
+
+    :ivar data: Raw data in this chunk for a standard TDMS channel.
+    :ivar scaler_data: A dictionary of scaler data in this segment for
+        DAQmx raw data. Keys are the scaler id and values are data arrays.
+    """
+
+    def __init__(self, data, scaler_data):
+        self.data = data
+        self.scaler_data = scaler_data
+
+    def __len__(self):
+        if self.data is not None:
+            return len(self.data)
+        elif self.scaler_data is not None:
+            return next(len(d) for d in self.scaler_data.values())
+        return 0
+
+    @staticmethod
+    def empty():
+        return RawChannelDataChunk(None, None)
+
+    @staticmethod
+    def channel_data(data):
+        return RawChannelDataChunk(data, None)
+
+    @staticmethod
+    def scaler_data(data):
+        return RawChannelDataChunk(None, data)
+
+
+def fromfile(file, dtype, count, *args, **kwargs):
+    """Read from any file-like object into a numpy array"""
+
+    itemsize = np.dtype(dtype).itemsize
+    buffer = np.zeros(count * itemsize, np.uint8)
+    bytes_read = -1
+    offset = 0
+    while bytes_read != 0:
+        bytes_read = file.readinto(buffer[offset:])
+        offset += bytes_read
+    rounded_bytes = (offset // itemsize) * itemsize
+    buffer = buffer[:rounded_bytes]
+    buffer.dtype = dtype
+    return buffer
+
+
+def read_interleaved_segment_bytes(f, bytes_per_row, num_values):
+    """ Read a segment of interleaved data as rows of bytes
+    """
+    number_bytes = bytes_per_row * num_values
+    combined_data = fromfile(f, dtype=np.uint8, count=number_bytes)
+
+    try:
+        # Reshape, so that one row is all bytes for all objects
+        combined_data = combined_data.reshape(-1, bytes_per_row)
+    except ValueError:
+        # Probably incomplete segment at the end => try to clip data
+        crop_len = (combined_data.shape[0] // bytes_per_row)
+        crop_len *= bytes_per_row
+        log.warning("Cropping data from %d to %d bytes to match segment "
+                    "size derived from channels",
+                    combined_data.shape[0], crop_len)
+        combined_data = combined_data[:crop_len].reshape(-1, bytes_per_row)
+    return combined_data
+
+
+def data_chunk_to_channel_chunk(data_chunk, channel_path):
+    try:
+        return data_chunk.channel_data[channel_path]
+    except KeyError:
+        return RawChannelDataChunk.empty()

nptdms/channel_data.py

@@ -0,0 +1,205 @@
+""" Responsible for storing data read from TDMS files
+"""
+
+import tempfile
+import numpy as np
+
+from nptdms import types
+from nptdms.timestamp import TimestampArray
+from nptdms.log import log_manager
+
+log = log_manager.get_logger(__name__)
+
+
+def get_data_receiver(obj, num_values, raw_timestamps, memmap_dir=None):
+    """Return a new channel data receiver to use for the given TDMS object
+
+    :param obj: TDMS channel object to receive data for
+    :param num_values: Number of values to be stored
+    :param raw_timestamps: Whether to store timestamp data as raw TDMS timestamps or a numpy datetime64 array
+    :param memmap_dir: Optional directory to store memory map files,
+        or None to not use memory map files
+    """
+    if obj.data_type == types.DaqMxRawData:
+        return DaqmxDataReceiver(obj, num_values, memmap_dir)
+
+    if obj.data_type == types.TimeStamp:
+        return TimestampDataReceiver(obj, num_values, raw_timestamps, memmap_dir)
+
+    if obj.data_type is None or obj.data_type.nptype is None:
+        return ListDataReceiver(obj)
+
+    return NumpyDataReceiver(obj, num_values, memmap_dir)
+
+
+class ListDataReceiver(object):
+    """Simple list based data receiver for objects that don't have a
+        corresponding numpy data type
+
+       :ivar data: List of data points
+    """
+
+    def __init__(self, channel):
+        """Initialise new data receiver for a TDMS object
+        """
+        if channel.data_type == types.String:
+            self._dtype = np.dtype('O')
+        else:
+            self._dtype = None
+        self._data = []
+        self.scaler_data = {}
+
+    def append_data(self, data):
+        """Append data from a segment
+        """
+        self._data.extend(data)
+
+    @property
+    def data(self):
+        dtype = self._dtype if self._dtype is not None else np.dtype('V8')
+        return np.array(self._data, dtype=dtype)
+
+
+class NumpyDataReceiver(object):
+    """Receives data for a TDMS object and stores it in a numpy array
+
+    :ivar data: Data that has been read for the object
+    """
+
+    def __init__(self, obj, num_values, memmap_dir=None):
+        """Initialise data receiver backed by a numpy array
+
+        :param obj: Object to store data for
+        :param num_values: Number of values to be stored
+        :param memmap_dir: Optional directory to store memory map files in.
+        """
+
+        self.path = obj.path
+        self.data = _new_numpy_array(
+            obj.data_type.nptype, num_values, memmap_dir)
+        self.scaler_data = {}
+        self._data_insert_position = 0
+        log.debug("Allocated %d sample slots for %s", len(self.data), obj.path)
+
+    def append_data(self, new_data):
+        """Update the object data with a new array of data"""
+
+        log.debug("Adding %d data points to data for %s", len(new_data), self.path)
+        start_pos = self._data_insert_position
+        end_pos = self._data_insert_position + len(new_data)
+        self.data[start_pos:end_pos] = new_data
+        self._data_insert_position += len(new_data)
+
+
+class DaqmxDataReceiver(object):
+    """Receives raw scaler data for a DAQmx object and stores it in numpy
+    arrays
+
+    :ivar scaler_data: Dictionary mapping from scaler id to data for a scaler
+    """
+
+    def __init__(self, obj, num_values, memmap_dir=None):
+        """Initialise data receiver for DAQmx backed by a numpy array
+
+        :param obj: Object to store data for
+        :param memmap_dir: Optional directory to store memory mmap files in.
+        """
+
+        self.path = obj.path
+        self.data = None
+        self.scaler_data = {}
+        self._scaler_insert_positions = {}
+        for scaler_id, scaler_type in obj.scaler_data_types.items():
+            self.scaler_data[scaler_id] = _new_numpy_array(
+                scaler_type.nptype, num_values, memmap_dir)
+            self._scaler_insert_positions[scaler_id] = 0
+
+    def append_scaler_data(self, scale_id, new_data):
+        """Append new DAQmx scaler data read from a segment
+        """
+
+        log.debug("Adding %d data points for object %s, scaler %d",
+                  len(new_data), self.path, scale_id)
+        data_array = self.scaler_data[scale_id]
+        start_pos = self._scaler_insert_positions[scale_id]
+        end_pos = start_pos + len(new_data)
+        data_array[start_pos:end_pos] = new_data
+        self._scaler_insert_positions[scale_id] += len(new_data)
+
+
+class TimestampDataReceiver(object):
+    """Receives timestamp data for a TDMS object and stores it in a numpy array
+
+    :ivar data: Data that has been read for the object as either a TimestampArray
+        or datetime64 array depending on whether raw_timestamps is True or False
+    """
+
+    def __init__(self, obj, num_values, raw_timestamps=False, memmap_dir=None):
+        """Initialise timestamp data receiver backed by a numpy array
+
+        :param obj: Object to store data for
+        :param num_values: Number of values to be stored
+        :param raw_timestamps: Whether to store data as raw TDMS timestamps or a numpy datetime64 array
+        :param memmap_dir: Optional directory to store memory map files in.
+        """
+
+        self.path = obj.path
+        self._raw_timestamps = raw_timestamps
+        if raw_timestamps:
+            byte_array = _new_numpy_array(np.dtype('uint8'), num_values * 16, memmap_dir)
+            dtype = np.dtype([('second_fractions', 'uint64'), ('seconds', 'int64')])
+            self.data = TimestampArray(byte_array.view(dtype))
+        else:
+            self.data = _new_numpy_array(np.dtype('datetime64[us]'), num_values, memmap_dir)
+        self.scaler_data = {}
+        self._data_insert_position = 0
+        log.debug("Allocated %d sample slots for %s", len(self.data), obj.path)
+
+    def append_data(self, new_data):
+        """Update the object data with a new array of data"""
+
+        log.debug("Adding %d data points to data for %s", len(new_data), self.path)
+        start_pos = self._data_insert_position
+        end_pos = self._data_insert_position + len(new_data)
+        if self._raw_timestamps:
+            # Need to be careful about potential endianness mismatch, so order of fields can differ
+            self.data['seconds'][start_pos:end_pos] = new_data['seconds']
+            self.data['second_fractions'][start_pos:end_pos] = new_data['second_fractions']
+        else:
+            self.data[start_pos:end_pos] = new_data.as_datetime64()
+        self._data_insert_position += len(new_data)
+
+
+class RawDataSlice(object):
+    def __init__(self, data, scaler_data):
+        self.data = data
+        self.scaler_data = scaler_data
+
+
+def slice_raw_data(raw_data, offset, length=None):
+    if offset == 0 and length is None:
+        return raw_data
+    end = None if length is None else offset + length
+    data = None if raw_data.data is None else raw_data.data[offset:end]
+    scaler_data = dict(
+        (scale_id, scaler_data[offset:end]) for (scale_id, scaler_data) in raw_data.scaler_data.items())
+    return RawDataSlice(data, scaler_data)
+
+
+def _new_numpy_array(dtype, num_values, memmap_dir=None):
+    """Initialise a new numpy array for data
+
+    :param dtype: Numpy data type for array
+    :param num_values: Capacity required
+    :param memmap_dir: Optional directory to store memory mmap files
+    """
+    if memmap_dir:
+        memmap_file = tempfile.NamedTemporaryFile(
+            mode='w+b', prefix="nptdms_", dir=memmap_dir)
+        return np.memmap(
+            memmap_file.file,
+            mode='w+',
+            shape=(num_values,),
+            dtype=dtype)
+
+    return np.zeros(num_values, dtype=dtype)

nptdms/common.py

@@ -0,0 +1,99 @@
+from itertools import zip_longest
+
+
+toc_properties = {
+    'kTocMetaData': (1 << 1),
+    'kTocRawData': (1 << 3),
+    'kTocDAQmxRawData': (1 << 7),
+    'kTocInterleavedData': (1 << 5),
+    'kTocBigEndian': (1 << 6),
+    'kTocNewObjList': (1 << 2)
+}
+
+
+class ObjectPath(object):
+    """ Represents the path of an object in a TDMS file
+
+        :ivar group: Group name or None for the root object
+        :ivar channel: Channel name or None for the root object or a group objecct
+    """
+    def __init__(self, *path_components):
+        self.group = None
+        self.channel = None
+        if len(path_components) > 0:
+            self.group = path_components[0]
+        if len(path_components) > 1:
+            self.channel = path_components[1]
+        if len(path_components) > 2:
+            raise ValueError("Object path may only have up to two components")
+        self._path = _components_to_path(self.group, self.channel)
+
+    @property
+    def is_root(self):
+        return self.group is None
+
+    @property
+    def is_group(self):
+        return self.group is not None and self.channel is None
+
+    @property
+    def is_channel(self):
+        return self.channel is not None
+
+    def group_path(self):
+        """ For channel paths, returns the path of the channel's group as a string
+        """
+        return _components_to_path(self.group, None)
+
+    @staticmethod
+    def from_string(path_string):
+        components = list(_path_components(path_string))
+        return ObjectPath(*components)
+
+    def __str__(self):
+        """ String representation of the object path
+        """
+        return self._path
+
+
+def _path_components(path):
+    """ Generator that yields components within an object path
+    """
+    # Iterate over each character and the next character
+    chars = zip_longest(path, path[1:])
+    try:
+        # Iterate over components
+        while True:
+            char, next_char = next(chars)
+            if char != '/':
+                raise ValueError("Invalid path, expected \"/\"")
+            elif next_char is not None and next_char != "'":
+                raise ValueError("Invalid path, expected \"'\"")
+            else:
+                # Consume "'" or raise StopIteration if at the end
+                next(chars)
+            component = []
+            # Iterate over characters in component name
+            while True:
+                char, next_char = next(chars)
+                if char == "'" and next_char == "'":
+                    component += "'"
+                    # Consume second "'"
+                    next(chars)
+                elif char == "'":
+                    yield "".join(component)
+                    break
+                else:
+                    component += char
+    except StopIteration:
+        return
+
+
+def _components_to_path(group, channel):
+    components = []
+    if group is not None:
+        components.append(group)
+    if channel is not None:
+        components.append(channel)
+    return ('/' + '/'.join(
+        ["'" + c.replace("'", "''") + "'" for c in components]))