dclab 0.67.0__cp314-cp314t-macosx_10_13_x86_64.whl

dclab/rtdc_dataset/fmt_tdms/event_contour.py

@@ -0,0 +1,264 @@
+"""Class for efficiently handling contour data"""
+import numbers
+import sys
+import warnings
+
+import numpy as np
+
+from ...features import inert_ratio
+
+from .exc import ContourIndexingError
+
+
+class ContourVerificationWarning(UserWarning):
+    pass
+
+
+class ContourColumn(object):
+    def __init__(self, rtdc_dataset):
+        """A wrapper for ContourData that takes into account event offsets
+
+        Event offsets appear when the first event that is recorded in the
+        tdms files does not have a corresponding contour in the contour
+        text file.
+        """
+        fname = self.find_contour_file(rtdc_dataset)
+        if fname is None:
+            self.identifier = None
+        else:
+            if sys.version_info[0] == 2:
+                self.identifier = str(fname).decode("utf-8")
+            else:
+                self.identifier = str(fname)
+        if fname is not None:
+            self._contour_data = ContourData(fname)
+            self._initialized = False
+        else:
+            self._contour_data = []
+            # prevent `determine_offset` to be called
+            self._initialized = True
+        self.frame = rtdc_dataset["frame"]
+        if "image" in rtdc_dataset:
+            self.image_shape = rtdc_dataset["image"].shape[1:]
+        else:
+            self.image_shape = None
+        # if they are set, these features are used for verifying the contour
+        self.pxfeat = {}
+
+        if "area_msd" in rtdc_dataset:
+            self.pxfeat["area_msd"] = rtdc_dataset["area_msd"]
+        if "pixel size" in rtdc_dataset.config["imaging"]:
+            px_size = rtdc_dataset.config["imaging"]["pixel size"]
+            for key in ["pos_x", "pos_y", "size_x", "size_y"]:
+                if key not in rtdc_dataset.features_innate:
+                    # abort
+                    self.pxfeat.clear()
+                    break
+                self.pxfeat[key] = rtdc_dataset[key] / px_size
+
+        self.event_offset = 0
+        self._length = None
+        self.dtype = np.int16
+
+    def __getitem__(self, idx):
+        if not isinstance(idx, numbers.Integral):
+            raise NotImplementedError(
+                "The RTDC_TDMS data handler does not support indexing with "
+                "anything else than scalar integers. Please convert your data "
+                "to the .rtdc file format!")
+
+        if not self._initialized:
+            self.determine_offset()
+        idnew = idx-self.event_offset
+        cdata = None
+        if idnew < 0:
+            # No contour data
+            cdata = np.zeros((2, 2), dtype=int)
+        else:
+            # Assign contour based on stored frame index
+            frame_ist = self.frame[idx]
+            # Do not only check the exact frame, but +/- 2 events around it
+            for idn in [idnew, idnew-1, idnew+1, idnew-2, idnew+2]:
+                # check frame
+                try:
+                    frame_soll = self._contour_data.get_frame(idn)
+                except IndexError:
+                    # reached end of file
+                    continue
+                if np.allclose(frame_soll, frame_ist, rtol=0):
+                    cdata = self._contour_data[idn]
+                    break
+        if cdata is None and self.image_shape and self.pxfeat:
+            # The frame is wrong, but the contour might be correct.
+            # We check that by verifying several features.
+            cdata2 = self._contour_data[idnew]
+            cont = np.zeros((self.image_shape[1], self.image_shape[0]))
+            cont[cdata2[:, 0], cdata2[:, 1]] = True
+            mm = inert_ratio.cont_moments_cv(cdata2)
+
+            if (np.allclose(self.pxfeat["size_x"][idx],
+                            np.ptp(cdata2[:, 0]) + 1,
+                            rtol=0, atol=1e-5)
+                and np.allclose(self.pxfeat["size_y"][idx],
+                                np.ptp(cdata2[:, 1]) + 1,
+                                rtol=0, atol=1e-5)
+                and np.allclose(mm["m00"],
+                                self.pxfeat["area_msd"][idx],
+                                rtol=0, atol=1e-5)
+                # atol=6 for positions, because the original positions
+                # are computed from the convex contour, which would be
+                # computed using cv2.convexHull(cdata2).
+                and np.allclose(self.pxfeat["pos_x"][idx],
+                                mm["m10"]/mm["m00"],
+                                rtol=0, atol=6)
+                and np.allclose(self.pxfeat["pos_y"][idx],
+                                mm["m01"]/mm["m00"],
+                                rtol=0, atol=6)):
+                cdata = cdata2
+
+        if cdata is None:
+            # No idea what went wrong, but we make the beste guess and
+            # issue a warning.
+            cdata = self._contour_data[idnew]
+            frame_c = self._contour_data.get_frame(idnew)
+            warnings.warn(
+                "Couldn't verify contour {} in {}".format(idx, self.identifier)
+                + " (frame index {})!".format(frame_c),
+                ContourVerificationWarning
+            )
+        return cdata
+
+    def __len__(self):
+        if self._length is None:
+            length = len(self._contour_data)
+            if length:
+                if not self._initialized:
+                    self.determine_offset()
+                length += self.event_offset
+            self._length = length
+        return self._length
+
+    @property
+    def shape(self):
+        return len(self), np.nan, 2
+
+    def determine_offset(self):
+        """Determines the offset of the contours w.r.t. other data columns
+
+        Notes
+        -----
+        - the "frame" column of `rtdc_dataset` is compared to
+          the first contour in the contour text file to determine an
+          offset by one event
+        - modifies the property `event_offset` and sets `_initialized`
+          to `True`
+        """
+        # In case of regular RTDC, the first contour is
+        # missing. In case of fRTDC, it is there, so we
+        # might have an offset. We find out if the first
+        # contour frame is missing by comparing it to
+        # the "frame" column of the rtdc dataset.
+        fref = self._contour_data.get_frame(0)
+        f0 = self.frame[0]
+        f1 = self.frame[1]
+        # Use allclose to avoid float/integer comparison problems
+        if np.allclose(fref, f0, rtol=0):
+            self.event_offset = 0
+        elif np.allclose(fref, f1, rtol=0):
+            self.event_offset = 1
+        else:
+            msg = "Contour data has unknown offset (frame {})!".format(fref)
+            raise ContourIndexingError(msg)
+        self._initialized = True
+
+    @staticmethod
+    def find_contour_file(rtdc_dataset):
+        """Tries to find a contour file that belongs to an RTDC dataset
+
+        Returns None if no contour file is found.
+        """
+        cont_id = rtdc_dataset.path.stem
+        cands = [c.name for c in rtdc_dataset._fdir.glob("*_contours.txt")]
+        cands = sorted(cands)
+        # Search for perfect matches, e.g.
+        # - M1_0.240000ul_s.tdms
+        # - M1_0.240000ul_s_contours.txt
+        for c1 in cands:
+            if c1.startswith(cont_id):
+                cfile = rtdc_dataset._fdir / c1
+                break
+        else:
+            # Search for M* matches with most overlap, e.g.
+            # - M1_0.240000ul_s.tdms
+            # - M1_contours.txt
+            for c2 in cands:
+                if (c2.split("_")[0] == rtdc_dataset._mid):
+                    # Do not confuse with M10_contours.txt
+                    cfile = rtdc_dataset._fdir / c2
+                    break
+            else:
+                cfile = None
+        return cfile
+
+
+class ContourData(object):
+    def __init__(self, fname):
+        """Access an MX_contour.txt as a dictionary
+
+        Initialize this class with a *_contour.txt file.
+        The individual contours can be accessed like a
+        list (enumerated from 0 on).
+        """
+        self._initialized = False
+        self.filename = fname
+        self._length = None
+
+    def __getitem__(self, idx):
+        cont = self.data[idx]
+        cont = cont.strip()
+        cont = cont.replace(")", "")
+        cont = cont.replace("(", "")
+        cont = cont.replace("(", "")
+        cont = cont.replace("\n", ",")
+        cont = cont.replace("   ", " ")
+        cont = cont.replace("  ", " ")
+        if len(cont) > 1:
+            _frame, cont = cont.split(" ", 1)
+            cont = cont.strip(" ,")
+            data = np.fromstring(cont, sep=",", dtype=np.uint16).reshape(-1, 2)
+            return data
+
+    def __len__(self):
+        if self._length is None:
+            self._length = len(self.data)
+        return self._length
+
+    def _index_file(self):
+        """Open and index the contour file
+
+        This function populates the internal list of contours
+        as strings which will be available as `self.data`.
+        """
+        with self.filename.open() as fd:
+            data = fd.read()
+
+        ident = "Contour in frame"
+        self._data = data.split(ident)[1:]
+        self._initialized = True
+
+    @property
+    def data(self):
+        """Access self.data
+        If `self._index_file` has not been computed before, this
+        property will cause it to do so.
+        """
+        if not self._initialized:
+            self._index_file()
+        return self._data
+
+    def get_frame(self, idx):
+        """Return the frame number of a contour"""
+        cont = self.data[idx]
+        # previously was split using " ", but "(" is more general
+        frame = int(cont.strip().split("(", 1)[0])
+        return frame

dclab/rtdc_dataset/fmt_tdms/event_image.py

@@ -0,0 +1,220 @@
+"""
+Class for efficiently handling image/video data
+"""
+import numbers
+import pathlib
+import sys
+import warnings
+
+import imageio
+import numpy as np
+
+from .exc import (InvalidVideoFileError, CorruptFrameWarning,
+                  InitialFrameMissingWarning, SlowVideoWarning)
+
+
+ISWIN = sys.platform.startswith("win")
+
+
+class ImageColumn(object):
+    def __init__(self, rtdc_dataset):
+        """A wrapper for ImageMap that takes into account event offsets
+
+        Event offsets appear when the first event that is recorded in the
+        tdms files does not have a corresponding cell image in the video
+        file.
+        """
+        fname = self.find_video_file(rtdc_dataset)
+        self.identifier = fname
+        if fname is not None:
+            self._image_data = ImageMap(fname)
+        else:
+            self._image_data = []
+        conf = rtdc_dataset.config
+        self.event_offset = int(conf["fmt_tdms"]["video frame offset"])
+        self.video_file = fname
+        self._shape = None
+        self.dtype = np.uint8
+
+    def __getitem__(self, idx):
+        if not isinstance(idx, numbers.Integral):
+            raise NotImplementedError(
+                "The RTDC_TDMS data handler does not support indexing with "
+                "anything else than scalar integers. Please convert your data "
+                "to the .rtdc file format!")
+
+        idnew = int(idx-self.event_offset)
+        if idnew < 0:
+            # No data - show a dummy image instead
+            warnings.warn("Frame {} in {} ".format(idnew, self.identifier)
+                          + "is not defined; replacing with dummy image!",
+                          InitialFrameMissingWarning)
+            cdata = self.dummy
+        else:
+            if hasattr(imageio.plugins.ffmpeg, "CannotReadFrameError"):
+                # imageio<2.5.0
+                excs = IndexError, imageio.plugins.ffmpeg.CannotReadFrameError
+            else:
+                # imageio>=2.5.0
+                excs = IndexError
+            try:
+                cdata = self._image_data[idnew]
+            except excs:
+                # The avi is corrupt. Return a dummy image.
+                warnings.warn("Frame {} in {} ".format(idnew, self.identifier)
+                              + "is corrupt; replacing with dummy image!",
+                              CorruptFrameWarning)
+                cdata = self.dummy
+        return cdata
+
+    def __len__(self):
+        length = len(self._image_data)
+        if length:
+            length = length + self.event_offset
+        return length
+
+    @property
+    def dummy(self):
+        """Returns a dummy image"""
+        cdata = np.zeros(self.shape[1:], dtype=np.uint8)
+        return cdata
+
+    @property
+    def shape(self):
+        if self._shape is None:
+            f0 = self._image_data[0].shape
+            self._shape = len(self), f0[0], f0[1]
+        return self._shape
+
+    @staticmethod
+    def find_video_file(rtdc_dataset):
+        """Tries to find a video file that belongs to an RTDC dataset
+
+        Returns None if no video file is found.
+        """
+        video = None
+        if rtdc_dataset._fdir.exists():
+            # Cell images (video)
+            videos = [v.name for v in rtdc_dataset._fdir.glob("*.avi")]
+            # Filter videos according to measurement number
+            meas_id = rtdc_dataset._mid
+            videos = [v for v in videos if v.split("_")[0] == meas_id]
+            videos.sort()
+            if len(videos) != 0:
+                # Defaults to first avi file
+                video = videos[0]
+                # g/q video file names. q comes first.
+                for v in videos:
+                    if v.endswith("imag.avi"):
+                        video = v
+                        break
+                    # add this here, because fRT-DC measurements also contain
+                    # videos ..._proc.avi
+                    elif v.endswith("imaq.avi"):
+                        video = v
+                        break
+        if video is None:
+            return None
+        else:
+            vpath = rtdc_dataset._fdir / video
+            if vpath.stat().st_size < 64:
+                # the video file is empty
+                raise InvalidVideoFileError("Bad video '{}'!".format(vpath))
+            else:
+                return vpath
+
+
+class ImageMap(object):
+    def __init__(self, fname):
+        """Access a video file of an RT-DC dataset
+
+        Initialize this class with a video file.
+        """
+        fname = pathlib.Path(fname)
+        self._length = None
+        # video handle:
+        self._cap = None
+        # filename
+        if not fname.exists():
+            raise OSError("file does not exist: {}".format(fname))
+        self.filename = fname
+
+    def __del__(self):
+        if self._cap is not None and hasattr(self._cap, "_proc"):
+            if ISWIN and self._cap._proc is not None:
+                # This is a workaround for windows when pytest fails due
+                # to "OSError: [WinError 6] The handle is invalid",
+                # which is somehow related to the fact that "_proc.kill()"
+                # must be called twice (in "close()" and in this case) in
+                # order to terminate the process and due to the fact the
+                # we are not using the with-statement in combination
+                # with imageio.get_reader().
+                self._cap._proc.kill()
+            self._cap.close()
+
+    def __getitem__(self, idx):
+        """Returns the requested frame from the video in gray scale"""
+        cap = self.video_handle
+        try:
+            cellimg = cap.get_data(idx)
+        except IndexError:
+            self._get_image_workaround_seek(idx)
+        else:
+            if np.all(cellimg == 0):
+                cellimg = self._get_image_workaround_seek(idx)
+        # Convert to grayscale
+        if len(cellimg.shape) == 3:
+            cellimg = np.array(cellimg[:, :, 0])
+        return cellimg
+
+    def __len__(self):
+        """Returns the length of the video or `True` if the length cannot be
+        determined.
+        """
+        if self._length is None:
+            cap = self.video_handle
+            if hasattr(cap, "count_frames"):  # imageio>=2.5.0
+                length = cap.count_frames()
+            else:  # imageio<2.5.0
+                try:
+                    length = len(cap)
+                except TypeError:
+                    # length is set to inf (it would generally still be
+                    # possible to rescue a limted number of frames, but
+                    # other parts of the dataset are probably also broken
+                    # (aborted measurement), so for the sake of simplicity
+                    # we stop here)
+                    raise InvalidVideoFileError(
+                        "Video file has unknown length '{}'!".format(
+                            self.filename))
+            self._length = length
+        return self._length
+
+    def _get_image_workaround_seek(self, idx):
+        """Same as __getitem__ but seek through the video beforehand
+
+        This is a workaround for an all-zero image returned by `imageio`.
+        """
+        cap = self.video_handle
+        mult = 50
+        # get the first frame to be on the safe side
+        cap.get_data(101)  # frame above 100 (don't know why)
+        cap.get_data(0)  # seek to zero
+        for ii in range(idx//mult):
+            cap.get_data(ii*mult)
+        final = cap.get_data(idx)
+        if not np.all(final == 0):
+            # This means we succeeded
+            warnings.warn("Seeking video file does not work, used workaround "
+                          + "which is slow!", SlowVideoWarning)
+        return final
+
+    @property
+    def video_handle(self):
+        if self._cap is None:
+            try:
+                self._cap = imageio.get_reader(self.filename)
+            except OSError:
+                raise InvalidVideoFileError(
+                    "Broken video file '{}'!".format(self.filename))
+        return self._cap

dclab/rtdc_dataset/fmt_tdms/event_mask.py

@@ -0,0 +1,62 @@
+"""Class for on-the-fly conversion of contours to masks"""
+import numbers
+
+import numpy as np
+import scipy.ndimage as ndi
+
+
+class MaskColumn(object):
+    def __init__(self, rtdc_dataset):
+        """Computes mask from contour data"""
+        self.contour = rtdc_dataset["contour"]
+        self.image = rtdc_dataset["image"]
+        self.identifier = self.contour.identifier
+        self.config = rtdc_dataset.config
+        self._shape = None
+        self._img_shape_cache = None
+        self.dtype = np.bool_
+
+    def __getitem__(self, idx):
+        if not isinstance(idx, numbers.Integral):
+            raise NotImplementedError(
+                "The RTDC_TDMS data handler does not support indexing with "
+                "anything else than scalar integers. Please convert your data "
+                "to the .rtdc file format!")
+
+        mask = np.zeros(self._img_shape, dtype=bool)
+        conti = self.contour[idx]
+        mask[conti[:, 1], conti[:, 0]] = True
+        ndi.binary_fill_holes(mask, output=mask)
+        return mask
+
+    def __len__(self):
+        if self._img_shape != (0, 0):
+            lc = len(self.contour)
+        else:
+            lc = 0
+        return lc
+
+    @property
+    def _img_shape(self):
+        if self._img_shape_cache is None:
+            """Shape of one event image"""
+            cfgim = self.config["imaging"]
+            if self.image:
+                # get shape from image column
+                event_image_shape = self.image.shape[1:]
+            elif "roi size x" in cfgim and "roi size y" in cfgim:
+                # get shape from config (this is less reliable than getting
+                # the shape from the image; there were measurements with
+                # wrong config keys)
+                event_image_shape = (cfgim["roi size y"], cfgim["roi size x"])
+            else:
+                # no shape available
+                event_image_shape = (0, 0)
+            self._img_shape_cache = event_image_shape
+        return self._img_shape_cache
+
+    @property
+    def shape(self):
+        if self._shape is None:
+            self._shape = len(self), self._img_shape[0], self._img_shape[1]
+        return self._shape

dclab/rtdc_dataset/fmt_tdms/event_trace.py

@@ -0,0 +1,146 @@
+"""Handling fluorescence trace data"""
+import pathlib
+import warnings
+
+from nptdms import TdmsFile
+import numpy as np
+
+from ... import definitions as dfn
+
+from . import naming
+from .exc import InvalidTDMSFileFormatError, MultipleSamplesPerEventFound
+
+
+class TraceColumn(object):
+    def __init__(self, rtdc_dataset):
+        """Prepares everything but does not load the trace data yet
+
+        The trace data is loaded when __getitem__, __len__, or __iter__
+        are called. This saves time and memory when the trace data is
+        not needed at all, e.g. for batch processing with DCscope.
+        """
+        self._trace = None
+        self.mname = rtdc_dataset.path
+        self.identifier = self.mname
+        self.dtype = np.int16
+
+    def __getitem__(self, trace_key):
+        if trace_key not in dfn.FLUOR_TRACES:
+            msg = "Unknown fluorescence trace key: {}".format(trace_key)
+            raise ValueError(msg)
+        return self.trace.__getitem__(trace_key)
+
+    def __len__(self):
+        return self.trace.__len__()
+
+    def __iter__(self):
+        return self.trace.__iter__()
+
+    def __repr__(self):
+        tname = TraceColumn.find_trace_file(self.mname)
+        if self._trace is None:
+            addstr = "not loaded into memory"
+        else:
+            addstr = "loaded into memory"
+
+        if tname is None:
+            rep = "No trace data available!"
+        else:
+            rep = "Fluorescence trace data from file {}, <{}>".format(tname,
+                                                                      addstr)
+        return rep
+
+    def keys(self):
+        return self.trace.keys()
+
+    @property
+    def shape(self):
+        key0 = sorted(self.keys())[0]
+        return len(self), len(self[key0]), len(self[key0][0])
+
+    @property
+    def trace(self):
+        """Initializes the trace data"""
+        if self._trace is None:
+            self._trace = self.load_trace(self.mname)
+        return self._trace
+
+    @staticmethod
+    def load_trace(mname):
+        """Loads the traces and returns them as a dictionary
+
+        Currently, only loading traces from tdms files is supported.
+        This forces us to load the full tdms file into memory which
+        takes some time.
+        """
+        tname = TraceColumn.find_trace_file(mname)
+
+        # Initialize empty trace dictionary
+        trace = {}
+
+        if tname is None:
+            pass
+        elif tname.suffix == ".tdms":
+            # Again load the measurement tdms file.
+            # This might increase memory usage, but it is cleaner
+            # when looking at code structure.
+            mdata = TdmsFile(str(mname))
+            try:
+                sampleids = mdata["Cell Track"]["FL1index"].data
+            except KeyError:
+                raise InvalidTDMSFileFormatError(
+                    "No 'FL1index' column in '{}'!".format(tname))
+
+            # Check that sample IDs are always incremented with same
+            # sample size.
+            samples_per_event = np.unique(np.diff(sampleids))
+            if len(samples_per_event) > 1:
+                # This means the length of the fluorescence trace is not
+                # a constant. According to Philipp, this means the trace
+                # cannot be used.
+                warnings.warn("Ignoring trace data of '{}' ".format(tname)
+                              + "due to multiple values for samples per "
+                              + "event: {}".format(samples_per_event),
+                              MultipleSamplesPerEventFound)
+            else:
+                # Load the trace data. The traces file is usually larger than
+                # the measurement file.
+                tdata = TdmsFile(str(tname))
+                for trace_key in dfn.FLUOR_TRACES:
+                    group, ch = naming.tr_data_map[trace_key]
+                    try:
+                        trdat = tdata[group][ch].data
+                    except KeyError:
+                        pass
+                    else:
+                        if trdat is not None and trdat.size != 0:
+                            # Split the input trace data into equally-spaced
+                            # sections (we already tested that sampleids is
+                            # equally-spaced).
+                            spe = sampleids[1] - sampleids[0]
+                            if (trdat.size % spe) == 0:
+                                # this is the ideal case
+                                trace_array = trdat.reshape(
+                                    trdat.size // spe, -1)
+                            else:
+                                # this is bad, but we allow it
+                                trace_array = np.split(trdat, trdat.size//spe)
+                            trace[trace_key] = trace_array
+
+        return trace
+
+    @staticmethod
+    def find_trace_file(mname):
+        """Tries to find the traces tdms file name
+
+        Returns None if no trace file is found.
+        """
+        mname = pathlib.Path(mname)
+        tname = None
+
+        if mname.exists():
+            cand = mname.with_name(mname.name[:-5] + "_traces.tdms")
+            if cand.exists():
+                tname = cand
+
+        return tname