dcnum 0.13.2__py3-none-any.whl → 0.23.1__py3-none-any.whl

dcnum/feat/feat_background/bg_sparse_median.py

@@ -1,24 +1,21 @@
 import logging
-import multiprocessing as mp
 import queue
 import time
 
 import numpy as np
 from scipy import ndimage
 
-from .base import Background
-
-logger = logging.getLogger(__name__)
+from ...read import HDF5Data
 
+from .base import mp_spawn, Background
 
-# All subprocesses should use 'spawn' to avoid issues with threads
-# and 'fork' on POSIX systems.
-mp_spawn = mp.get_context('spawn')
+logger = logging.getLogger(__name__)
 
 
 class BackgroundSparseMed(Background):
     def __init__(self, input_data, output_path, kernel_size=200,
                  split_time=1., thresh_cleansing=0, frac_cleansing=.8,
+                 offset_correction=True,
                  compress=True, num_cpus=None):
         """Sparse median background correction with cleansing
 
@@ -61,6 +58,21 @@ class BackgroundSparseMed(Background):
             Fraction between 0 and 1 indicating how many background images
             must still be present after cleansing (in case the cleansing
             factor is too large). Set to 1 to disable cleansing altogether.
+        offset_correction: bool
+            The sparse median background correction produces one median
+            image for multiple input frames (BTW this also leads to very
+            efficient data storage with HDF5 data compression filters). In
+            case the input frames are subject to frame-by-frame brightness
+            variations (e.g. flickering of the illumination source), it
+            is useful to have an offset value per frame that can then be
+            used in a later step to perform a more accurate background
+            correction. This offset is computed here by taking a 20px wide
+            slice from each frame (where the channel wall is located)
+            and computing the median therein relative to the computed
+            background image. The data are written to the "bg_off" feature
+            in the output file alongside "image_bg". To obtain the
+            corrected background image, add "image_bg" and "bg_off".
+            Set this to False if you don't need the "bg_off" feature.
         compress: bool
             Whether to compress background data. Set this to False
             for faster processing.
@@ -76,7 +88,15 @@ class BackgroundSparseMed(Background):
             kernel_size=kernel_size,
             split_time=split_time,
             thresh_cleansing=thresh_cleansing,
-            frac_cleansing=frac_cleansing)
+            frac_cleansing=frac_cleansing,
+            offset_correction=offset_correction,
+        )
+
+        if kernel_size > len(self.input_data):
+            logger.warning(
+                f"The kernel size {kernel_size} is too large for input data"
+                f"size {len(self.input_data)}. Setting it to input data size!")
+            kernel_size = len(self.input_data)
 
         #: kernel size used for median filtering
         self.kernel_size = kernel_size
@@ -86,31 +106,34 @@ class BackgroundSparseMed(Background):
         self.thresh_cleansing = thresh_cleansing
         #: keep at least this many background images from the series
         self.frac_cleansing = frac_cleansing
+        #: offset/flickering correction
+        self.offset_correction = offset_correction
 
         # time axis
         self.time = None
         if self.h5in is not None:
-            if "time" in self.h5in["events"]:
+            hd = HDF5Data(self.h5in)
+            if "time" in hd:
                 # use actual time from dataset
-                self.time = self.h5in["/events/time"][:]
+                self.time = hd["time"][:]
                 self.time -= self.time[0]
-            elif "imaging:frame rate" in self.h5in.attrs:
-                fr = self.h5in.attrs["imaging:frame rate"]
-                if "frame" in self.h5in["/events"]:
+            elif "imaging:frame rate" in hd.meta:
+                fr = hd.meta["imaging:frame rate"]
+                if "frame" in hd:
                     # compute time from frame rate and frame numbers
-                    self.time = self.h5in["/events/frame"] / fr
+                    self.time = hd["frame"] / fr
                     self.time -= self.time[0]
                 else:
                     # compute time using frame rate (approximate)
-                    dur = self.event_count / fr * 1.5
+                    dur = self.image_count / fr * 1.5
                     logger.info(f"Approximating duration: {dur/60:.1f}min")
-                    self.time = np.linspace(0, dur, self.event_count,
+                    self.time = np.linspace(0, dur, self.image_count,
                                             endpoint=True)
         if self.time is None:
             # No HDF5 file or no information therein; Make an educated guess.
-            dur = self.event_count / 3600 * 1.5
+            dur = self.image_count / 3600 * 1.5
             logger.info(f"Guessing duration: {dur/60:.1f}min")
-            self.time = np.linspace(0, dur, self.event_count,
+            self.time = np.linspace(0, dur, self.image_count,
                                     endpoint=True)
 
         #: duration of the measurement
@@ -149,11 +172,11 @@ class BackgroundSparseMed(Background):
         #: queue for median computation jobs
         self.queue = mp_spawn.Queue()
         #: list of workers (processes)
-        self.workers = [MedianWorkerSingle(self.queue,
-                                           self.worker_counter,
-                                           self.shared_input_raw,
-                                           self.shared_output_raw,
-                                           self.kernel_size)
+        self.workers = [WorkerSparseMed(self.queue,
+                                        self.worker_counter,
+                                        self.shared_input_raw,
+                                        self.shared_output_raw,
+                                        self.kernel_size)
                         for _ in range(self.num_cpus)]
         [w.start() for w in self.workers]
 
@@ -172,11 +195,13 @@ class BackgroundSparseMed(Background):
                           kernel_size: int = 200,
                           split_time: float = 1.,
                           thresh_cleansing: float = 0,
-                          frac_cleansing: float = .8):
+                          frac_cleansing: float = .8,
+                          offset_correction: bool = True,
+                          ):
         """Initialize user-defined properties of this class
 
         This method primarily exists so that the CLI knows which
-        keyword arguements can be passed to this class.
+        keyword arguments can be passed to this class.
 
         Parameters
         ----------
@@ -194,6 +219,21 @@ class BackgroundSparseMed(Background):
             Fraction between 0 and 1 indicating how many background images
             must still be present after cleansing (in case the cleansing
             factor is too large). Set to 1 to disable cleansing altogether.
+        offset_correction: bool
+            The sparse median background correction produces one median
+            image for multiple input frames (BTW this also leads to very
+            efficient data storage with HDF5 data compression filters). In
+            case the input frames are subject to frame-by-frame brightness
+            variations (e.g. flickering of the illumination source), it
+            is useful to have an offset value per frame that can then be
+            used in a later step to perform a more accurate background
+            correction. This offset is computed here by taking a 20px wide
+            slice from each frame (where the channel wall is located)
+            and computing the median therein relative to the computed
+            background image. The data are written to the "bg_off" feature
+            in the output file alongside "image_bg". To obtain the
+            corrected background image, add "image_bg" and "bg_off".
+            Set this to False if you don't need the "bg_off" feature.
         """
         assert kernel_size > 0
         assert split_time > 0
@@ -206,10 +246,7 @@ class BackgroundSparseMed(Background):
 
         # Compute initial background images (populates self.bg_images)
         for ii, ti in enumerate(self.step_times):
-            print(f"Computing background {ii / self.step_times.size:.0%}",
-                  end="\r", flush=True)
             self.process_second(ii, ti)
-        print("Computing background 100%    ", flush=True)
 
         if self.frac_cleansing != 1:
             # The following algorithm finds background images that contain
@@ -271,16 +308,16 @@ class BackgroundSparseMed(Background):
                     f"`thresh_cleansing` or `frac_cleansing`. The new "
                     f"threshold is {thresh_fact / thresh}.")
 
-            logger.info(f"Removed {frac_remove:.2%} of the background series")
+            logger.info(f"Cleansed {frac_remove:.2%}")
             step_times = self.step_times[used]
             bg_images = self.bg_images[used]
         else:
-            logger.info("Background series cleansing disabled.")
+            logger.info("Background series cleansing disabled")
             step_times = self.step_times
             bg_images = self.bg_images
 
         # Assign each frame to a certain background index
-        bg_idx = np.zeros(self.event_count, dtype=int)
+        bg_idx = np.zeros(self.image_count, dtype=int)
         idx0 = 0
         idx1 = None
         for ii in range(len(step_times)):
@@ -292,26 +329,38 @@ class BackgroundSparseMed(Background):
             # Fill up remainder of index array with last entry
             bg_idx[idx1:] = ii
 
+        self.image_proc.value = 1
+
         # Write background data
         pos = 0
         step = 1000
-        while pos < self.event_count:
-            stop = min(pos + step, self.event_count)
+        while pos < self.image_count:
+            stop = min(pos + step, self.image_count)
             cur_slice = slice(pos, stop)
-            self.h5out["events/image_bg"][cur_slice] = \
-                bg_images[bg_idx[cur_slice]]
+            cur_bg_data = bg_images[bg_idx[cur_slice]]
+            self.writer.store_feature_chunk("image_bg", cur_bg_data)
+            if self.offset_correction:
+                # Record background offset correction "bg_off". We take a
+                # slice of 20px from the top of the image (there are normally
+                # no events here, only the channel walls are visible).
+                sh, sw = self.input_data.shape[1:]
+                roi_full = (slice(None), slice(0, 20), slice(0, sw))
+                roi_cur = (cur_slice, slice(0, 20), slice(0, sw))
+                val_bg = np.mean(cur_bg_data[roi_full], axis=(1, 2))
+                val_dat = np.mean(self.input_data[roi_cur], axis=(1, 2))
+                # background image = image_bg + bg_off
+                self.writer.store_feature_chunk("bg_off", val_dat - val_bg)
             pos += step
 
-    def process_second(self, ii, second):
+    def process_second(self,
+                       ii: int,
+                       second: float | int):
         idx_start = np.argmin(np.abs(second - self.time))
         idx_stop = idx_start + self.kernel_size
-        if idx_stop >= self.event_count:
-            # If `idx_stop` is always greater than `self.event_count`,
-            # then `diff` is always greater than zero.
-            diff = idx_stop - self.event_count
-            idx_stop -= diff
-            idx_start -= diff
-            assert idx_start >= 0
+        if idx_stop >= self.image_count:
+            idx_stop = self.image_count
+            idx_start = max(0, idx_stop - self.kernel_size)
+        assert idx_stop - idx_start == self.kernel_size
 
         # The following is equivalent to, but faster than:
         # self.bg_images[ii] = np.median(self.input_data[idx_start:idx_stop],
@@ -344,12 +393,14 @@ class BackgroundSparseMed(Background):
 
         self.bg_images[ii] = self.shared_output.reshape(self.image_shape)
 
+        self.image_proc.value = idx_stop / self.image_count
+
 
-class MedianWorkerSingle(mp_spawn.Process):
+class WorkerSparseMed(mp_spawn.Process):
     def __init__(self, job_queue, counter, shared_input, shared_output,
                  kernel_size, *args, **kwargs):
         """Worker process for median computation"""
-        super(MedianWorkerSingle, self).__init__(*args, **kwargs)
+        super(WorkerSparseMed, self).__init__(*args, **kwargs)
         self.queue = job_queue
         self.queue.cancel_join_thread()
         self.counter = counter

dcnum/feat/feat_brightness/__init__.py

@@ -1,3 +1,4 @@
 # flake8: noqa: F401
+"""Feature computation: brightness-based features"""
 from .bright_all import brightness_features
 from .common import brightness_names

dcnum/feat/feat_brightness/bright_all.py

@@ -4,16 +4,38 @@ import numpy as np
 from .common import brightness_names
 
 
-def brightness_features(image,
-                        mask,
-                        image_bg=None,
-                        image_corr=None):
+def brightness_features(image: np.ndarray[np.uint8],
+                        mask: np.ndarray[np.bool_],
+                        image_bg: np.ndarray[np.uint8] = None,
+                        image_corr: np.ndarray[np.int16] = None,
+                        bg_off: float = None,
+                        ):
+    """Compute brightness features
+
+    Parameters
+    ----------
+    image: np.ndarray
+        2D array of "image" of shape (H, W)
+    mask: np.ndarray
+        3D array containing the N masks of shape (N, H, W)
+    image_bg: np.ndarray
+        2D array of "image_bg" of shape (H, W), required for computing
+        the "bg_med" feature.
+    image_corr: np.ndarray
+        2D array of (image - image_bg), which can be optionally passed
+        to this method. If not given, will be computed.
+    bg_off: float
+        Systematic offset value for correcting the brightness of the
+        background data which has an effect on "bright_bc_avg",
+        "bright_perc_10", "bright_perc_90", and "bg_med" (`bg_off` is
+        generated by sparsemed background correction).
+    """
     mask = np.array(mask, dtype=bool)
     size = mask.shape[0]
 
     br_dict = {}
-    for key in brightness_names:
-        br_dict[key] = np.full(size, np.nan, dtype=np.float64)
+    for mkey in brightness_names:
+        br_dict[mkey] = np.full(size, np.nan, dtype=np.float64)
 
     avg_sd = compute_avg_sd_masked_uint8(image, mask)
     br_dict["bright_avg"][:] = avg_sd[:, 0]
@@ -36,6 +58,19 @@ def brightness_features(image,
         br_dict["bright_perc_10"][:] = percentiles[:, 0]
         br_dict["bright_perc_90"][:] = percentiles[:, 1]
 
+    if bg_off is not None:
+        # subtract the background offset for all values that are computed
+        # from background-corrected images
+        for mkey in ["bright_bc_avg", "bright_perc_10", "bright_perc_90"]:
+            if mkey in br_dict:
+                br_dict[mkey] -= bg_off
+
+        # add the background offset to all values that were computed from
+        # the background only
+        for pkey in ["bg_med"]:
+            if pkey in br_dict:
+                br_dict[pkey] += bg_off
+
     return br_dict
 
 

dcnum/feat/feat_contour/__init__.py

@@ -0,0 +1,4 @@
+# flake8: noqa: F401
+"""Feature computation: OpenCV moments-based features"""
+from .moments import moments_based_features
+from .volume import volume_from_contours

dcnum/feat/{feat_moments/mt_legacy.py → feat_contour/moments.py}

@@ -2,11 +2,27 @@ import cv2
 import numpy as np
 
 
-from .ct_opencv import contour_single_opencv
-
-
-def moments_based_features(mask, pixel_size):
+from .contour import contour_single_opencv
+
+
+def moments_based_features(
+        mask: np.ndarray,
+        pixel_size: float,
+        ret_contour: bool = False,
+        ):
+    """Compute moment-based features for a mask image
+
+    Parameters
+    ----------
+    mask: np.ndarray
+        3D stack of 2D boolean mask images to analyze
+    pixel_size: float
+        pixel size of the mask image in µm
+    ret_contour: bool
+        whether to also return the raw contour
+    """
     assert pixel_size is not None and pixel_size != 0
+    raw_contours = []
 
     size = mask.shape[0]
 
@@ -42,9 +58,13 @@ def moments_based_features(mask, pixel_size):
     for ii in range(size):
         # raw contour
         cont_raw = contour_single_opencv(mask[ii])
-        if len(cont_raw.shape) < 2:
-            continue
-        if cv2.contourArea(cont_raw) == 0:
+        # only continue if the contour is valid
+        not_valid = len(cont_raw.shape) < 2 or cv2.contourArea(cont_raw) == 0
+
+        if ret_contour:
+            raw_contours.append(None if not_valid else cont_raw)
+
+        if not_valid:
             continue
 
         mu_raw = cv2.moments(cont_raw)
@@ -53,6 +73,7 @@ def moments_based_features(mask, pixel_size):
 
         # convex hull
         cont_cvx = np.squeeze(cv2.convexHull(cont_raw))
+
         mu_cvx = cv2.moments(cont_cvx)
         arc_cvx = np.float64(cv2.arcLength(cont_cvx, True))
 
@@ -110,7 +131,7 @@ def moments_based_features(mask, pixel_size):
         # specify validity
         valid[ii] = True
 
-    return {
+    data = {
         "area_msd": feat_area_msd,
         "area_ratio": feat_area_ratio,
         "area_um": feat_area_um,
@@ -131,3 +152,6 @@ def moments_based_features(mask, pixel_size):
         "tilt": feat_tilt,
         "valid": valid,
     }
+    if ret_contour:
+        data["contour"] = raw_contours
+    return data

dcnum/feat/feat_contour/volume.py

@@ -0,0 +1,174 @@
+from typing import List
+
+import numpy as np
+
+
+def volume_from_contours(
+        contour: List[np.ndarray],
+        pos_x: np.ndarray,
+        pos_y: np.ndarray,
+        pixel_size: float):
+    """Calculate the volume of a polygon revolved around an axis
+
+    The volume estimation assumes rotational symmetry.
+
+    Parameters
+    ----------
+    contour: list of ndarrays of shape (N,2)
+        One entry is a 2D array that holds the contour of an event
+    pos_x: float ndarray of length N
+        The x coordinate(s) of the centroid of the event(s) [µm]
+    pos_y: float ndarray of length N
+        The y coordinate(s) of the centroid of the event(s) [µm]
+    pixel_size: float
+        The detector pixel size in µm.
+
+    Returns
+    -------
+    volume: float ndarray
+        volume in um^3
+
+    Notes
+    -----
+    The computation of the volume is based on a full rotation of the
+    upper and the lower halves of the contour from which the
+    average is then used.
+
+    The volume is computed radially from the center position
+    given by (`pos_x`, `pos_y`). For sufficiently smooth contours,
+    such as densely sampled ellipses, the center position does not
+    play an important role. For contours that are given on a coarse
+    grid, as is the case for deformability cytometry, the center position
+    must be given.
+
+    References
+    ----------
+    - https://de.wikipedia.org/wiki/Kegelstumpf#Formeln
+    - Yields identical results to the Matlab script by Geoff Olynyk
+      <https://de.mathworks.com/matlabcentral/fileexchange/36525-volrevolve>`_
+    """
+    # results are stored in a separate array initialized with nans
+    v_avg = np.zeros_like(pos_x, dtype=np.float64) * np.nan
+
+    for ii in range(pos_x.shape[0]):
+        # If the contour has less than 4 pixels, the computation will fail.
+        # In that case, the value np.nan is already assigned.
+        cc = contour[ii]
+        if cc is not None and cc.shape[0] >= 4:
+            # Center contour coordinates with given centroid
+            contour_x = cc[:, 0] - pos_x[ii] / pixel_size
+            contour_y = cc[:, 1] - pos_y[ii] / pixel_size
+            # Switch to r and z to follow notation of vol_revolve
+            # (In RT-DC the axis of rotation is x, but for vol_revolve
+            # we need the axis vertically)
+            contour_r = contour_y
+            contour_z = contour_x
+
+            # Compute right volume
+            # Which points are at negative r-values (r<0)?
+            inx_neg = np.where(contour_r < 0)
+            # These points will be shifted up to r=0 directly on the z-axis
+            contour_right = np.copy(contour_r)
+            contour_right[inx_neg] = 0
+            vol_right = vol_revolve(r=contour_right,
+                                    z=contour_z,
+                                    point_scale=pixel_size)
+
+            # Compute left volume
+            # Which points are at positive r-values? (r>0)?
+            idx_pos = np.where(contour_r > 0)
+            # These points will be shifted down to y=0 to build an x-axis
+            contour_left = np.copy(contour_r)
+            contour_left[idx_pos] = 0
+            # Now we still have negative r values, but vol_revolve needs
+            # positive values, so we flip the sign...
+            contour_left[:] *= -1
+            # ... but in doing so, we have switched to clockwise rotation,
+            # and we need to pass the array in reverse order
+            vol_left = vol_revolve(r=contour_left[::-1],
+                                   z=contour_z[::-1],
+                                   point_scale=pixel_size)
+
+            # Compute the average
+            v_avg[ii] = (vol_right + vol_left) / 2
+
+    return {"volume": v_avg}
+
+
+def vol_revolve(r, z, point_scale=1.):
+    r"""Calculate the volume of a polygon revolved around the Z-axis
+
+    This implementation yields the same results as the volRevolve
+    Matlab function by Geoff Olynyk (from 2012-05-03)
+    https://de.mathworks.com/matlabcentral/fileexchange/36525-volrevolve.
+
+    The difference here is that the volume is computed using (a much
+    more approachable) implementation using the volume of a truncated
+    cone (https://de.wikipedia.org/wiki/Kegelstumpf).
+
+    .. math::
+
+      V = \frac{h \cdot \pi}{3} \cdot (R^2 + R \cdot r + r^2)
+
+    Where :math:`h` is the height of the cone and :math:`r` and
+    `R` are the smaller and larger radii of the truncated cone.
+
+    Each line segment of the contour resembles one truncated cone. If
+    the z-step is positive (counter-clockwise contour), then the
+    truncated cone volume is added to the total volume. If the z-step
+    is negative (e.g. inclusion), then the truncated cone volume is
+    removed from the total volume.
+
+    Parameters
+    ----------
+    r: 1d np.ndarray
+        radial coordinates (perpendicular to the z axis)
+    z: 1d np.ndarray
+        coordinate along the axis of rotation
+    point_scale: float
+        point size in your preferred units; The volume is multiplied
+        by a factor of `point_scale**3`.
+
+    Notes
+    -----
+    The coordinates must be given in counter-clockwise order,
+    otherwise the volume will be negative.
+    """
+    r = np.atleast_1d(r)
+    z = np.atleast_1d(z)
+
+    # make sure we have a closed contour
+    if (r[-1] != r[0]) or (z[-1] != z[0]):
+        # We have an open contour - close it.
+        r = np.resize(r, len(r) + 1)
+        z = np.resize(z, len(z) + 1)
+
+    rp = r[:-1]
+
+    # array of radii differences: R - r
+    dr = np.diff(r)
+    # array of height differences: h
+    dz = np.diff(z)
+
+    # If we expand the function in the doc string with
+    # dr = R - r and dz = h, then we get three terms for the volume
+    # (as opposed to four terms in Olynyk's script). Those three terms
+    # all resemble area slices multiplied by the z-distance dz.
+    a1 = 3 * rp ** 2
+    a2 = 3 * rp * dr
+    a3 = dr ** 2
+
+    # Note that the formula for computing the volume is symmetric
+    # with respect to r and R. This means that it does not matter
+    # which sign dr has (R and r are always positive). Since this
+    # algorithm assumes that the contour is ordered counter-clockwise,
+    # positive dz means adding to the contour while negative dz means
+    # subtracting from the contour (see test functions for more details).
+    # Conveniently so, dz only appears one time in this formula, so
+    # we can take the sign of dz as it is (Otherwise, we would have
+    # to take the absolute value of every truncated cone volume and
+    # multiply it by np.sign(dz)).
+    v = np.pi / 3 * dz * np.abs(a1 + a2 + a3)
+    vol = np.sum(v) * point_scale ** 3
+
+    return vol

dcnum/feat/feat_texture/__init__.py

@@ -1,2 +1,3 @@
 # flake8: noqa: F401
+"""Feature computation: Haralick texture features"""
 from .tex_all import haralick_names, haralick_texture_features

dcnum/feat/feat_texture/tex_all.py

@@ -6,6 +6,34 @@ from .common import haralick_names
 
 def haralick_texture_features(
         mask, image=None, image_bg=None, image_corr=None):
+    """Compute Haralick texture features
+
+    The following texture features are excluded
+
+    - feature 6 "Sum Average", which is equivalent to `2 * bright_bc_avg`
+      since dclab 0.44.0
+    - feature 10 "Difference Variance", because it has a functional
+      dependency on the offset value and since we do background correction,
+      we are not interested in it
+    - feature 14, because nobody is using it, it is not understood by
+      everyone what it actually is, and it is computationally expensive.
+
+    This leaves us with the following 11 texture features (22 if you count
+    avg and ptp):
+    https://earlglynn.github.io/RNotes/package/EBImage/Haralick-Textural-Features.html
+
+    - 1. `tex_asm`: (1) Angular Second Moment
+    - 2. `tex_con`: (2) Contrast
+    - 3. `tex_cor`: (3) Correlation
+    - 4. `tex_var`: (4) Variance
+    - 5. `tex_idm`: (5) Inverse Difference Moment
+    - 6. `tex_sva`: (7) Sum Variance
+    - 7. `tex_sen`: (8) Sum Entropy
+    - 8. `tex_ent`: (9) Entropy
+    - 9. `tex_den`: (11) Difference Entropy
+    - 10. `tex_f12`: (12) Information Measure of Correlation 1
+    - 11. `tex_f13`: (13) Information Measure of Correlation 2
+    """
     # make sure we have a boolean array
     mask = np.array(mask, dtype=bool)
     size = mask.shape[0]
@@ -22,7 +50,6 @@ def haralick_texture_features(
 
     for ii in range(size):
         # Haralick texture features
-        # https://gitlab.gwdg.de/blood_data_analysis/dcevent/-/issues/20
         # Preprocessing:
         # - create a copy of the array (don't edit `image_corr`)
         # - add grayscale values (negative values not supported)