msr-reader 0.2.1.dev0__tar.gz → 0.3.0__tar.gz

msr_reader-0.3.0/.gitignore

@@ -0,0 +1,6 @@
+*.egg-info
+dist
+.idea
+*__pycache__
+msr-test-files
+.DS_Store

{msr_reader-0.2.1.dev0 → msr_reader-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: msr-reader
-Version: 0.2.1.dev0
+Version: 0.3.0
 Summary: Pure Python reader for MSR/OBF (Imspector) image data
 Project-URL: Homepage, https://github.com/hoerlteam/msr-reader
 Project-URL: Bug Tracker, https://github.com/hoerlteam/msr-reader/issues
@@ -41,6 +41,23 @@ with OBFFile('file_path.msr') as f:
     idx = 0
     img = f.read_stack(idx) # read stack with index idx into numpy array
 
+    # multi-channel (or confocal + STED) data is acquired into separate stacks by Imspector.
+    # stack_groups groups stacks with same series number in the stack name and identical geometry.
+    # every stack belongs to exactly one group, unmatched stacks
+    # (e.g. line profiles, rescue_info maps) form single-member groups.
+    groups = f.stack_groups # list of StackGroup
+    img = f.read_stack_group(0) # read all "channels", stacked along a new first axis: e.g. (C, Z, Y, X)
+
+    # stacks without names (sometimes found in old files) can be grouped by geometry alone:
+    # with OBFFile('file_path.msr', group_unnamed_stacks=True) as f:
+
+    # Lightbox files use hierarchical stack names ('Overview 1/Image 1/14:03:01/STAR RED'):
+    # for .obf files, grouping by the path up to the last component (the
+    # acquisition) and the last component (the channel name) is enabled by
+    # default. for .msr files (Imspector), enable it explicitly:
+    # with OBFFile('file_path.msr', group_hierarchical_stacks=True) as f:
+    # pass group_hierarchical_stacks=False to disable it for .obf files.
+
     # metadata
     stack_shapes = f.shapes # list of stack shapes, including stack and dimension names
     pixel_sizes = f.pixel_sizes # like shapes, but with pixel sizes (unit: meters)

{msr_reader-0.2.1.dev0 → msr_reader-0.3.0}/README.md

@@ -25,6 +25,23 @@ with OBFFile('file_path.msr') as f:
     idx = 0
     img = f.read_stack(idx) # read stack with index idx into numpy array
 
+    # multi-channel (or confocal + STED) data is acquired into separate stacks by Imspector.
+    # stack_groups groups stacks with same series number in the stack name and identical geometry.
+    # every stack belongs to exactly one group, unmatched stacks
+    # (e.g. line profiles, rescue_info maps) form single-member groups.
+    groups = f.stack_groups # list of StackGroup
+    img = f.read_stack_group(0) # read all "channels", stacked along a new first axis: e.g. (C, Z, Y, X)
+
+    # stacks without names (sometimes found in old files) can be grouped by geometry alone:
+    # with OBFFile('file_path.msr', group_unnamed_stacks=True) as f:
+
+    # Lightbox files use hierarchical stack names ('Overview 1/Image 1/14:03:01/STAR RED'):
+    # for .obf files, grouping by the path up to the last component (the
+    # acquisition) and the last component (the channel name) is enabled by
+    # default. for .msr files (Imspector), enable it explicitly:
+    # with OBFFile('file_path.msr', group_hierarchical_stacks=True) as f:
+    # pass group_hierarchical_stacks=False to disable it for .obf files.
+
     # metadata
     stack_shapes = f.shapes # list of stack shapes, including stack and dimension names
     pixel_sizes = f.pixel_sizes # like shapes, but with pixel sizes (unit: meters)

{msr_reader-0.2.1.dev0 → msr_reader-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "msr-reader"
-version = "0.2.1-dev"
+version = "0.3.0"
 authors = [
   { name="David Hörl", email="hoerldavid@gmail.com" },
 ]

msr_reader-0.3.0/src/msr_reader/__init__.py

@@ -0,0 +1,2 @@
+from .obffile import OBFFile, OBFHeader, OBFStackHeader, OBFStackFooter, OBFSIFraction, OBFSIUnit, StackGroup, StackSizes
+from .xml_metadata import imspector_xml_to_dict

{msr_reader-0.2.1.dev0 → msr_reader-0.3.0}/src/msr_reader/obffile.py

@@ -1,4 +1,7 @@
+import os
+import re
 import struct
+import warnings
 from collections import namedtuple
 from io import BytesIO
 from zlib import decompress
@@ -11,21 +14,64 @@ MAGIC_STACK_HEADER = b'OMAS_BF_STACK\n\xff\xff'
 
 OBFHeader = namedtuple('OBFHeader', ['format_version', 'description', 'first_stack_position', 'metadata_position', 'metadata'])
 
-OBFStackHeader = namedtuple('OBFStackHeader',
+class OBFStackHeader(namedtuple('OBFStackHeader',
     ['stack_version', 'size', 'length', 'offset', 'dtype', 'compressed', 'name',
-     'description', 'data_position', 'data_length', 'next_stack_position'])
+     'description', 'data_position', 'reserved', 'data_length', 'next_stack_position'])):
 
-OBFSIFraction = namedtuple('OBFSIFraction', ['numerator', 'denominator'])
-OBFSIUnit = namedtuple('OBFSIUnit', ['meters', 'kilograms', 'seconds', 'amperes', 'kelvin', 'moles',
-                            'candela', 'radian', 'steradian', 'scale_factor'])
+    @property
+    def numpy_dtype(self):
+        '''
+        numpy dtype (np.dtype instance, like arr.dtype) of the stack data. note
+        that dtype holds the raw OMAS_DT flag from the file (including the
+        complex bit and, for RGB types, the number of samples per pixel), which
+        is not fully captured by a numpy dtype. raises ValueError for
+        unsupported flags.
+        '''
+        return np.dtype(_parse_dtype(self.dtype)[0])
 
-OBFStackFooter = namedtuple('OBFStackFooter', ['size', 'has_col_positions', 'has_col_labels', 'metadata', 'si_value', 'si_dimensions',
+OBFSIFraction = namedtuple('OBFSIFraction', ['numerator', 'denominator'])
+class OBFSIUnit(namedtuple('OBFSIUnit', ['meters', 'kilograms', 'seconds', 'amperes', 'kelvin', 'moles',
+                            'candela', 'radian', 'steradian', 'scale_factor'])):
+    '''
+    an SI unit: the 9 base/supplementary units with rational exponents
+    (OBFSIFraction each, numerator 0 = unused), multiplied by scale_factor,
+    i.e. unit = scale_factor * m^p1 * kg^p2 * s^p3 * ...
+    '''
+
+    # symbols matching the exponent field order above
+    _SYMBOLS = ('m', 'kg', 's', 'A', 'K', 'mol', 'cd', 'rad', 'sr')
+
+    def __str__(self):
+        parts = []
+        for symbol, fraction in zip(self._SYMBOLS, self[:9]):
+            if fraction.numerator == 0:
+                continue
+            if fraction.denominator == 1:
+                parts.append(symbol if fraction.numerator == 1 else f'{symbol}^{fraction.numerator}')
+            else:
+                parts.append(f'{symbol}^({fraction.numerator}/{fraction.denominator})')
+        unit = ' * '.join(parts) if parts else 'dimensionless'
+        if self.scale_factor != 1:
+            unit += f' * {self.scale_factor:g}'
+        return unit
+
+    # make notebooks show the pretty form, not the namedtuple field wall
+    __repr__ = __str__
+
+OBFStackFooter = namedtuple('OBFStackFooter', ['size', 'has_col_positions', 'has_col_labels', 'col_positions', 'col_labels', 'metadata', 'si_value', 'si_dimensions',
                                                'flush_positions', 'flush_block_size', 'tag_dictionary', 'stack_end_disk',
                                                'min_format_version', 'stack_end_used_disk', 'samples_written', 'num_chunk_positions',
                                                'dimension_labels'])
 
 StackSizes = namedtuple('StackSizes', ['name', 'sizes', 'dimension_names'])
 
+# pattern of default Imspector stack names: a user-settable channel name
+# followed by the series number of the acquisition, e.g. 'STAR RED_CONF {0}'
+STACK_NAME_PATTERN = re.compile(r'^(?P<base>.+) \{(?P<series>\d+)\}$')
+
+StackGroup = namedtuple('StackGroup', ['series', 'channel_names', 'indices',
+                                       'shape', 'dtype', 'pixel_sizes', 'dimension_names'])
+
 def _read_file_header(fd):
     header_fmt = '<10sLQL'
     magic_header, format_version, first_stack_pos, descr_len = struct.unpack(header_fmt, fd.read(struct.calcsize(header_fmt)))
@@ -59,14 +105,14 @@ def _read_stack_header(fd, position):
     offset = struct.unpack(f'<{MAXIMAL_NUMBER_OF_DIMENSIONS}d', fd.read(struct.calcsize(f'<{MAXIMAL_NUMBER_OF_DIMENSIONS}d')))[:rank]
 
     stack_header_fmt2 = '<LLLLLQQQ'
-    data_type, compression_type, compression_level, name_len, descr_len, _, data_len, next_stack_pos = struct.unpack(stack_header_fmt2, fd.read(struct.calcsize(stack_header_fmt2)))
-    
+    data_type, compression_type, compression_level, name_len, descr_len, reserved, data_len, next_stack_pos = struct.unpack(stack_header_fmt2, fd.read(struct.calcsize(stack_header_fmt2)))
+
     name = fd.read(name_len).decode('utf-8')
     descr = fd.read(descr_len).decode('utf-8')
 
     data_pos = fd.tell()
 
-    return OBFStackHeader(version, size, length, offset, data_type, compression_type>0, name, descr, data_pos, data_len, next_stack_pos)
+    return OBFStackHeader(version, size, length, offset, data_type, compression_type>0, name, descr, data_pos, reserved, data_len, next_stack_pos)
 
 def _parse_dtype(dtype_flags: int):
 
@@ -116,7 +162,7 @@ def _parse_dtype(dtype_flags: int):
 
     return dtype, samples_per_pixel
 
-def _read_stack(fd, stack_header: OBFStackHeader):
+def _read_stack(fd, stack_header: OBFStackHeader, samples_written=None):
     fd.seek(stack_header.data_position)
     buffer = fd.read(stack_header.data_length)
 
@@ -126,6 +172,26 @@ def _read_stack(fd, stack_header: OBFStackHeader):
 
     dtype, samples_per_pixel = _parse_dtype(stack_header.dtype)
 
+    # reserved == 0 marks a data-less stack: no data on disk, the array is
+    # filled with zeros by convention (see file format spec)
+    if stack_header.reserved == 0:
+        warnings.warn(f'stack {stack_header.name!r} is data-less, the returned data is all zeros')
+
+    # a measurement ended early only writes samples_written samples (footer V6+),
+    # by convention the missing data is zero (see file format spec), so pad short
+    # buffers up to the full stack size (and drop any excess padding)
+    num_samples = int(np.prod(stack_header.size)) * samples_per_pixel
+    expected_bytes = num_samples * np.dtype(dtype).itemsize
+    if len(buffer) < expected_bytes:
+        buffer += b'\x00' * (expected_bytes - len(buffer))
+    elif len(buffer) > expected_bytes:
+        buffer = buffer[:expected_bytes]
+
+    # samples_written == 0 means the full stack is on disk (see spec)
+    if samples_written and samples_written < num_samples:
+        warnings.warn(f'stack {stack_header.name!r} was truncated, only {samples_written} of '
+                      f'{num_samples} samples were written, the remainder is zero-padded')
+
     return np.frombuffer(buffer, dtype=dtype).reshape(stack_header.size[::-1] + (samples_per_pixel, )).squeeze()
 
 def _read_tag_dict(buffer):
@@ -151,8 +217,26 @@ def _to_si_unit(unpack_output):
     return OBFSIUnit(*unit_fractions, unpack_output[-1])
     
 
+def _empty_stack_footer(stack_header: OBFStackHeader):
+    # stack footers only exist from stack version 1 on. version-0 stacks carry no
+    # footer, labels or tag dictionary on disk, so return empty/default values.
+    # dimension_labels are padded to the stack rank so they stay aligned with
+    # the stack axes (they are zipped together in OBFFile.shapes/pixel_sizes).
+    return OBFStackFooter(
+        size=None, has_col_positions=None, has_col_labels=None,
+        col_positions=[None] * len(stack_header.size), col_labels=[None] * len(stack_header.size),
+        metadata='',
+        si_value=None, si_dimensions=None, flush_positions=[], flush_block_size=None,
+        tag_dictionary={}, stack_end_disk=None, min_format_version=None,
+        stack_end_used_disk=None, samples_written=None, num_chunk_positions=None,
+        dimension_labels=[''] * len(stack_header.size))
+
 def _read_stack_footer(fd, stack_header: OBFStackHeader):
 
+    # version-0 stacks are written without any footer
+    if stack_header.stack_version < 1:
+        return _empty_stack_footer(stack_header)
+
     # seek to footer start (after binary data)
     fd.seek(stack_header.data_position + stack_header.data_length)
 
@@ -212,12 +296,36 @@ def _read_stack_footer(fd, stack_header: OBFStackHeader):
         labels.append(s.decode('utf8'))
 
     if stack_header.stack_version >= 1:
-        # TODO: handle col_positions, col_labels, chunked storage
-        if any(has_col_labels) or any(has_col_positions):
-            raise ValueError('col_label and col_position parsing not implemented yet')
+        # TODO: handle chunked storage
         if num_chunk_positions is not None and num_chunk_positions > 0:
             raise ValueError('chunked storage not implemented yet')
 
+        # column positions: for each axis with has_col_positions[i] != 0 an array
+        # of res[i] float64 values follows.
+        # entries are None for axes without positions.
+        # NOTE: we assume all position arrays come before the column label arrays
+        # TODO: needs testing on actual data!
+        col_positions = []
+        for i, has_pos in enumerate(has_col_positions):
+            if has_pos:
+                col_positions.append(np.frombuffer(fd.read(8 * stack_header.size[i]), dtype='<f8'))
+            else:
+                col_positions.append(None)
+
+        # column labels: for each axis with has_col_labels[i] != 0 an array of
+        # res[i] strings follows (same encoding as the dimension labels above).
+        # entries are None for axes without column labels.
+        col_labels = []
+        for i, has_lab in enumerate(has_col_labels):
+            if has_lab:
+                labels_i = []
+                for _ in range(stack_header.size[i]):
+                    (n,) = struct.unpack(f'<L', fd.read(struct.calcsize(f'<L')))
+                    labels_i.append(fd.read(n).decode('utf8'))
+                col_labels.append(labels_i)
+            else:
+                col_labels.append(None)
+
         metadata = fd.read(metadata_length).decode('utf-8')
     else:
         metadata = ""
@@ -234,7 +342,7 @@ def _read_stack_footer(fd, stack_header: OBFStackHeader):
     else:
         tag_dict = {}
 
-    return OBFStackFooter(size, has_col_positions, has_col_labels, metadata, si_value, si_dimensions, flush_positions,
+    return OBFStackFooter(size, has_col_positions, has_col_labels, col_positions, col_labels, metadata, si_value, si_dimensions, flush_positions,
                    flush_block_size, tag_dict, stack_end_disk, min_format_version, stack_end_used_disk,
                     samples_written, num_chunk_positions, labels)
 
@@ -246,10 +354,18 @@ class OBFFile:
     https://imspectordocs.readthedocs.io/en/latest/fileformat.html
     """
 
-    def __init__(self, path):
+    def __init__(self, path, group_unnamed_stacks=False, group_hierarchical_stacks=None):
         self.fd = open(path, 'rb')
+        self._group_unnamed_stacks = group_unnamed_stacks
+        if group_hierarchical_stacks is None:
+            # auto by file extension: Lightbox writes .obf files with
+            # hierarchical stack names, Imspector writes .msr files (OBF plus
+            # some undocumented extras like GUI state)
+            group_hierarchical_stacks = os.path.splitext(str(path))[1].lower() == '.obf'
+        self._group_hierarchical_stacks = group_hierarchical_stacks
         self._parse_headers()
         self.stack_names = [s.name for s in self.stack_headers]
+        self._stack_groups = None
 
     def _parse_headers(self):
         self.main_header = _read_file_header(self.fd)
@@ -265,7 +381,7 @@ class OBFFile:
             self.stack_footers.append(stack_footer)
 
     def read_stack(self, idx):
-        return _read_stack(self.fd, self.stack_headers[idx])
+        return _read_stack(self.fd, self.stack_headers[idx], self.stack_footers[idx].samples_written)
     
     @property
     def num_stacks(self):
@@ -316,6 +432,130 @@ class OBFFile:
         """
         return self.pixel_sizes[stack_idx].sizes
 
+    def _group_key(self, stack_idx):
+        # geometry part of the stack grouping key: shape, data type, pixel
+        # sizes and dimension labels.
+        header_i = self.stack_headers[stack_idx]
+        shapes_i = self.shapes[stack_idx]
+        pixel_sizes_i = self.pixel_sizes[stack_idx]
+        # round pixel sizes to a fixed number of significant digits to make the
+        # comparison robust against float noise: scientific notation rounds with
+        # relative precision, unlike round()/np.round() which round to decimal
+        # places (absolute precision) and would merge distinct pixel sizes
+        # around the nanometer scale (or collapse tiny values onto each other)
+        pixel_sizes_rounded = tuple(float(f'{p:.9e}') for p in pixel_sizes_i.sizes)
+        return (tuple(shapes_i.sizes), header_i.dtype, pixel_sizes_rounded, tuple(shapes_i.dimension_names))
+
+    @property
+    def stack_groups(self):
+        """
+        stacks grouped into channels: stacks acquired in the same series
+        (the number in curly braces in the stack name) with identical geometry
+        (shape, data type, pixel sizes, dimension labels) are considered
+        channels of the same acquisition. stacks that fit nowhere (e.g.
+        auxiliary stacks like rescue_info maps, or stacks without a parseable
+        name) form single-member groups. every stack is part of exactly one
+        group, so the groups are always a partition of all stacks.
+
+        with group_hierarchical_stacks enabled, names containing '/' (as
+        written by the Lightbox software, e.g. 'Overview 1/Image 1/14:03:01/
+        STAR RED') are grouped by the path up to the last component instead:
+        the path identifies the acquisition (the series), the last component
+        is the channel name. enabled by default for .obf files (Lightbox),
+        disabled for .msr files (Imspector); pass group_hierarchical_stacks
+        True/False to override the extension-based default.
+        """
+        if self._stack_groups is None:
+            self._stack_groups = self._compute_stack_groups()
+        return self._stack_groups
+
+    def _parse_stack_name(self, idx):
+        """
+        (series, channel name) parsed from the stack name, (None, None) for
+        stacks that are never treated as channels (auxiliary data like
+        rescue_info maps, or names that match no convention).
+
+        series is the acquisition identifier: the number in curly braces for
+        default Imspector names ('STAR RED_CONF {0}'), or the path up to the
+        last '/' component for hierarchical names if group_hierarchical_stacks
+        is enabled.
+        """
+        name = self.stack_names[idx]
+
+        # auxiliary data like ResCue switch-off maps is never treated as a channel
+        if 'rescue_info' in name:
+            return None, None
+
+        match = STACK_NAME_PATTERN.match(name)
+        if match is not None:
+            return int(match.group('series')), match.group('base')
+
+        if self._group_hierarchical_stacks:
+            path, _, channel = name.rpartition('/')
+            if path and channel:
+                return path, channel
+
+        return None, None
+
+    def _compute_stack_groups(self):
+        groups = []
+        key_to_group = {}
+        for idx in range(self.num_stacks):
+            name = self.stack_names[idx]
+            series, channel = self._parse_stack_name(idx)
+
+            if channel is not None:
+                key = (series,) + self._group_key(idx)
+            elif name == '' and self._group_unnamed_stacks:
+                # opt-in: group unnamed stacks by geometry alone (series unknown)
+                key = (None,) + self._group_key(idx)
+            else:
+                key = None
+
+            if key is None:
+                groups.append([idx])
+            elif key in key_to_group:
+                groups[key_to_group[key]].append(idx)
+            else:
+                key_to_group[key] = len(groups)
+                groups.append([idx])
+
+        return [self._make_stack_group(indices) for indices in groups]
+
+    def _make_stack_group(self, indices):
+        header_i = self.stack_headers[indices[0]]
+        shapes_i = self.shapes[indices[0]]
+        pixel_sizes_i = self.pixel_sizes[indices[0]]
+
+        series = None
+        channel_names = []
+        for idx in indices:
+            name = self.stack_names[idx]
+            s, channel = self._parse_stack_name(idx)
+            if channel is not None:
+                series = s
+                channel_names.append(channel)
+            elif name:
+                channel_names.append(name)
+            else:
+                channel_names.append(f'stack {idx}')
+
+        try:
+            dtype = np.dtype(_parse_dtype(header_i.dtype)[0])
+        except ValueError:
+            dtype = None
+
+        return StackGroup(series, channel_names, indices, tuple(shapes_i.sizes), dtype,
+                          list(pixel_sizes_i.sizes), list(shapes_i.dimension_names))
+
+    def read_stack_group(self, group_idx, axis=0):
+        """
+        read all stacks of a group, stacked along a new axis (default: first,
+        i.e. (C, Z, Y, X)). single-member groups get an axis of length one.
+        """
+        group = self.stack_groups[group_idx]
+        return np.stack([self.read_stack(i) for i in group.indices], axis=axis)
+
     def get_imspector_xml_metadata(self, stack_idx):
         """
         get XML metadata string of stack with index stack_idx
@@ -338,8 +578,9 @@ class OBFFile:
         get OME-XML metadata string
         """
 
-        # check that we have metadata (will be available through main header)
-        if 'ome_xml' in self.main_header.metadata:
+        # check that we have metadata (will be available through main header,
+        # but only from file format version 2 on)
+        if self.main_header.metadata and 'ome_xml' in self.main_header.metadata:
             ome_xml_metadata = self.main_header.metadata['ome_xml']
         else:
             ome_xml_metadata = ''