dcnum 0.17.0__py3-none-any.whl → 0.23.2__py3-none-any.whl

dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.17.0'
-__version_tuple__ = version_tuple = (0, 17, 0)
+__version__ = version = '0.23.2'
+__version_tuple__ = version_tuple = (0, 23, 2)

dcnum/feat/__init__.py

@@ -1,6 +1,6 @@
 # flake8: noqa: F401
 """Feature computation"""
-from . import feat_background, feat_brightness, feat_moments, feat_texture
+from . import feat_background, feat_brightness, feat_contour, feat_texture
 from .event_extractor_manager_thread import EventExtractorManagerThread
 from .queue_event_extractor import (
     QueueEventExtractor, EventExtractorThread, EventExtractorProcess

dcnum/feat/event_extractor_manager_thread.py

@@ -46,8 +46,8 @@ class EventExtractorManagerThread(threading.Thread):
             The queue the writer uses. We monitor this queue. If it
             fills up, we take a break.
         debug:
-            Whether to run in debugging mode which means more log
-            messages and only one thread (`num_workers` has no effect).
+            Whether to run in debugging mode which means only one
+            event extraction thread (`num_workers` has no effect).
         """
         super(EventExtractorManagerThread, self).__init__(
               name="EventExtractorManager", *args, **kwargs)
@@ -95,32 +95,41 @@ class EventExtractorManagerThread(threading.Thread):
         while True:
             # If the writer_dq starts filling up, then this could lead to
             # an oom-kill signal. Stall for the writer to prevent this.
-            ldq = len(self.writer_dq)
-            if ldq > 100:
-                stallsec = ldq / 100
+            if (ldq := len(self.writer_dq)) > 1000:
+                time.sleep(1)
+                ldq2 = len(self.writer_dq)
+                stall_time = max(0., (ldq2 - 200) / ((ldq - ldq2) or 1))
+                time.sleep(stall_time)
                 self.logger.warning(
-                    f"Stalling {stallsec:.1f}s for slow writer")
-                time.sleep(stallsec)
+                    f"Stalled {stall_time + 1:.1f}s for slow writer "
+                    f"({ldq} chunks queued)")
 
-            cur_slot = 0
             unavailable_slots = 0
+            found_free_slot = False
             # Check all slots for segmented labels
-            while True:
-                # - "e" there is data from the segmenter (the extractor
-                #   can take it and process it)
-                # - "s" the extractor processed the data and is waiting
-                #   for the segmenter
-                if self.slot_states[cur_slot] == "e":
-                    # The segmenter has something for us in this slot.
-                    break
-                else:
-                    # Try another slot.
-                    unavailable_slots += 1
-                    cur_slot = (cur_slot + 1) % num_slots
-                if unavailable_slots >= num_slots:
-                    # There is nothing to do, try to avoid 100% CPU
-                    unavailable_slots = 0
-                    time.sleep(.1)
+            while not found_free_slot:
+                # We sort the slots according to the slot chunks so that we
+                # always process the slot with the smallest slot chunk number
+                # first. Initially, the slot_chunks array is filled with
+                # zeros, but the segmenter fills up the slots with the lowest
+                # number first.
+                for cur_slot in np.argsort(self.slot_chunks):
+                    # - "e" there is data from the segmenter (the extractor
+                    #   can take it and process it)
+                    # - "s" the extractor processed the data and is waiting
+                    #   for the segmenter
+                    if self.slot_states[cur_slot] == "e":
+                        # The segmenter has something for us in this slot.
+                        found_free_slot = True
+                        break
+                    else:
+                        # Try another slot.
+                        unavailable_slots += 1
+                        cur_slot = (cur_slot + 1) % num_slots
+                    if unavailable_slots >= num_slots:
+                        # There is nothing to do, try to avoid 100% CPU
+                        unavailable_slots = 0
+                        time.sleep(.1)
 
             t1 = time.monotonic()
 
@@ -147,7 +156,7 @@ class EventExtractorManagerThread(threading.Thread):
             # We are done here. The segmenter may continue its deed.
             self.slot_states[cur_slot] = "w"
 
-            self.logger.debug(f"Extracted one chunk: {chunk}")
+            self.logger.debug(f"Extracted chunk {chunk} in slot {cur_slot}")
             self.t_count += time.monotonic() - t1
 
             chunks_processed += 1

dcnum/feat/feat_background/base.py

@@ -3,14 +3,12 @@ import functools
 import inspect
 import multiprocessing as mp
 import pathlib
-import uuid
 
 import h5py
-import numpy as np
 
 from ...meta import ppid
 from ...read import HDF5Data
-from ...write import create_with_basins, set_default_filter_kwargs
+from ...write import HDF5Writer, create_with_basins, set_default_filter_kwargs
 
 
 # All subprocesses should use 'spawn' to avoid issues with threads
@@ -64,8 +62,8 @@ class Background(abc.ABC):
 
         #: number of images in the input data
         self.image_count = None
-        #: number of images that have been processed
-        self.image_proc = mp_spawn.Value("L", 0)
+        #: fraction images that have been processed
+        self.image_proc = mp_spawn.Value("d", 0)
 
         #: HDF5Data instance for input data
         self.hdin = None
@@ -93,8 +91,6 @@ class Background(abc.ABC):
         else:
             self.input_data = input_data
 
-        #: unique identifier
-        self.name = str(uuid.uuid4())
         #: shape of event images
         self.image_shape = self.input_data[0].shape
         #: total number of events
@@ -109,25 +105,17 @@ class Background(abc.ABC):
             # "a", because output file already exists
             self.h5out = h5py.File(output_path, "a", libver="latest")
 
-        # Initialize background data
-        ds_kwargs = set_default_filter_kwargs(compression=compress)
-        h5bg = self.h5out.require_dataset(
-            "events/image_bg",
-            shape=self.input_data.shape,
-            dtype=np.uint8,
-            chunks=(min(100, self.image_count),
-                    self.image_shape[0],
-                    self.image_shape[1]),
-            **ds_kwargs,
+        # Initialize writer
+        self.writer = HDF5Writer(
+            obj=self.h5out,
+            ds_kwds=set_default_filter_kwargs(compression=compress),
         )
-        h5bg.attrs.create('CLASS', np.string_('IMAGE'))
-        h5bg.attrs.create('IMAGE_VERSION', np.string_('1.2'))
-        h5bg.attrs.create('IMAGE_SUBCLASS', np.string_('IMAGE_GRAYSCALE'))
 
     def __enter__(self):
         return self
 
     def __exit__(self, type, value, tb):
+        self.writer.close()
         # Close h5in and h5out
         if self.hdin is not None:  # we have an input file
             self.hdin.close()  # this closes self.h5in
@@ -142,7 +130,7 @@ class Background(abc.ABC):
         """Return a unique background pipeline identifier
 
         The pipeline identifier is universally applicable and must
-        be backwards-compatible (future versions of dcevent will
+        be backwards-compatible (future versions of dcnum will
         correctly acknowledge the ID).
 
         The segmenter pipeline ID is defined as::
@@ -197,15 +185,23 @@ class Background(abc.ABC):
         if self.image_count == 0:
             return 0.
         else:
-            return self.image_proc.value / self.image_count
+            return self.image_proc.value
 
     def process(self):
+        # Delete any old background data
+        for key in ["image_bg", "bg_off"]:
+            if key in self.h5out["events"]:
+                del self.h5out["events"][key]
+        # Perform the actual background computation
         self.process_approach()
         bg_ppid = self.get_ppid()
-        # Store pipeline information in the image_bg feature
-        self.h5out["events/image_bg"].attrs["dcnum ppid background"] = bg_ppid
-        self.h5out["events/image_bg"].attrs["dcnum ppid generation"] = \
-            ppid.DCNUM_PPID_GENERATION
+        # Store pipeline information in the image_bg/bg_off feature
+        for key in ["image_bg", "bg_off"]:
+            if key in self.h5out["events"]:
+                self.h5out[f"events/{key}"].attrs["dcnum ppid background"] = \
+                    bg_ppid
+                self.h5out[F"events/{key}"].attrs["dcnum ppid generation"] = \
+                    ppid.DCNUM_PPID_GENERATION
 
     @abc.abstractmethod
     def process_approach(self):

dcnum/feat/feat_background/bg_copy.py

@@ -4,22 +4,28 @@ from .base import Background
 
 
 class BackgroundCopy(Background):
-
     @staticmethod
     def check_user_kwargs():
         pass
 
-    def process_approach(self):
-        """Perform median computation on entire input data"""
+    def process(self):
+        """Copy input data to output dataset"""
         if self.h5in != self.h5out:
-            hin = self.hdin.image_bg.h5ds
-            if "image_bg" in self.h5out["events"]:
-                del self.h5out["events/image_bg"]
-            h5py.h5o.copy(src_loc=hin.parent.id,
-                          src_name=b"image_bg",
-                          dst_loc=self.h5out["events"].id,
-                          dst_name=b"image_bg",
-                          )
+            hin = self.hdin.h5
+            for feat in ["image_bg", "bg_off"]:
+                if feat in hin["events"]:
+                    h5py.h5o.copy(src_loc=hin["events"].id,
+                                  src_name=feat.encode("utf-8"),
+                                  dst_loc=self.h5out["events"].id,
+                                  dst_name=feat.encode("utf-8"),
+                                  )
 
         # set progress to 100%
-        self.image_proc.value = self.image_count
+        self.image_proc.value = 1
+
+    def process_approach(self):
+        # We do the copying in `process`, because we do not want to modify
+        # any metadata or delete datasets as is done in the base class.
+        # But we still have to implement this method, because it is an
+        # abstractmethod in the base class.
+        pass

dcnum/feat/feat_background/bg_roll_median.py

@@ -119,7 +119,7 @@ class BackgroundRollMed(Background):
         """Check user-defined properties of this class
 
         This method primarily exists so that the CLI knows which
-        keyword arguements can be passed to this class.
+        keyword arguments can be passed to this class.
 
         Parameters
         ----------
@@ -132,7 +132,8 @@ class BackgroundRollMed(Background):
             `kernel_size` will not increase computation speed. Larger
             values lead to a higher memory consumption.
         """
-        assert kernel_size > 0
+        assert kernel_size > 0, "kernel size must be positive number"
+        assert kernel_size % 2 == 0, "kernel size must be even number"
         assert batch_size > kernel_size
 
     def get_slices_for_batch(self, batch_index=0):
@@ -172,11 +173,18 @@ class BackgroundRollMed(Background):
         num_steps = int(np.ceil(self.image_count / self.batch_size))
         for ii in range(num_steps):
             self.process_next_batch()
-        # Set the remaining kernel_size median values to the last one
-        last_image = self.h5out["events/image_bg"][-self.kernel_size-1]
-        for ii in range(self.kernel_size):
-            self.h5out["events/image_bg"][self.image_count-ii-1] = last_image
-        self.image_proc.value = self.image_count
+
+        # Set the remaining median bg images to the last one.
+        num_remaining = (self.input_data.shape[0]
+                         - self.h5out["events/image_bg"].shape[0])
+        if num_remaining:
+            last_image = self.h5out["events/image_bg"][-1]
+            last_chunk = np.repeat(
+                last_image[np.newaxis],
+                num_remaining,
+                axis=0)
+            self.writer.store_feature_chunk("image_bg", last_chunk)
+        self.image_proc.value = 1
 
     def process_next_batch(self):
         """Process one batch of input data"""
@@ -208,12 +216,14 @@ class BackgroundRollMed(Background):
             # TODO:
             #  Do this in a different thread so workers can keep going
             #  and use a lock somewhere in case the disk is too slow.
-            self.h5out["events/image_bg"][cur_slice_out] = \
+            self.writer.store_feature_chunk(
+                "image_bg",
                 self.shared_output[:output_size].reshape(output_size,
-                                                         *self.image_shape)
+                                                         *self.image_shape),
+            )
 
         self.current_batch += 1
-        self.image_proc.value += self.batch_size
+        self.image_proc.value += self.batch_size / self.image_count
 
 
 class WorkerRollMed(mp_spawn.Process):

dcnum/feat/feat_background/bg_sparse_median.py

@@ -15,6 +15,7 @@ logger = logging.getLogger(__name__)
 class BackgroundSparseMed(Background):
     def __init__(self, input_data, output_path, kernel_size=200,
                  split_time=1., thresh_cleansing=0, frac_cleansing=.8,
+                 offset_correction=True,
                  compress=True, num_cpus=None):
         """Sparse median background correction with cleansing
 
@@ -57,6 +58,21 @@ class BackgroundSparseMed(Background):
             Fraction between 0 and 1 indicating how many background images
             must still be present after cleansing (in case the cleansing
             factor is too large). Set to 1 to disable cleansing altogether.
+        offset_correction: bool
+            The sparse median background correction produces one median
+            image for multiple input frames (BTW this also leads to very
+            efficient data storage with HDF5 data compression filters). In
+            case the input frames are subject to frame-by-frame brightness
+            variations (e.g. flickering of the illumination source), it
+            is useful to have an offset value per frame that can then be
+            used in a later step to perform a more accurate background
+            correction. This offset is computed here by taking a 20px wide
+            slice from each frame (where the channel wall is located)
+            and computing the median therein relative to the computed
+            background image. The data are written to the "bg_off" feature
+            in the output file alongside "image_bg". To obtain the
+            corrected background image, add "image_bg" and "bg_off".
+            Set this to False if you don't need the "bg_off" feature.
         compress: bool
             Whether to compress background data. Set this to False
             for faster processing.
@@ -72,7 +88,9 @@ class BackgroundSparseMed(Background):
             kernel_size=kernel_size,
             split_time=split_time,
             thresh_cleansing=thresh_cleansing,
-            frac_cleansing=frac_cleansing)
+            frac_cleansing=frac_cleansing,
+            offset_correction=offset_correction,
+        )
 
         if kernel_size > len(self.input_data):
             logger.warning(
@@ -88,6 +106,8 @@ class BackgroundSparseMed(Background):
         self.thresh_cleansing = thresh_cleansing
         #: keep at least this many background images from the series
         self.frac_cleansing = frac_cleansing
+        #: offset/flickering correction
+        self.offset_correction = offset_correction
 
         # time axis
         self.time = None
@@ -175,11 +195,13 @@ class BackgroundSparseMed(Background):
                           kernel_size: int = 200,
                           split_time: float = 1.,
                           thresh_cleansing: float = 0,
-                          frac_cleansing: float = .8):
+                          frac_cleansing: float = .8,
+                          offset_correction: bool = True,
+                          ):
         """Initialize user-defined properties of this class
 
         This method primarily exists so that the CLI knows which
-        keyword arguements can be passed to this class.
+        keyword arguments can be passed to this class.
 
         Parameters
         ----------
@@ -197,6 +219,21 @@ class BackgroundSparseMed(Background):
             Fraction between 0 and 1 indicating how many background images
             must still be present after cleansing (in case the cleansing
             factor is too large). Set to 1 to disable cleansing altogether.
+        offset_correction: bool
+            The sparse median background correction produces one median
+            image for multiple input frames (BTW this also leads to very
+            efficient data storage with HDF5 data compression filters). In
+            case the input frames are subject to frame-by-frame brightness
+            variations (e.g. flickering of the illumination source), it
+            is useful to have an offset value per frame that can then be
+            used in a later step to perform a more accurate background
+            correction. This offset is computed here by taking a 20px wide
+            slice from each frame (where the channel wall is located)
+            and computing the median therein relative to the computed
+            background image. The data are written to the "bg_off" feature
+            in the output file alongside "image_bg". To obtain the
+            corrected background image, add "image_bg" and "bg_off".
+            Set this to False if you don't need the "bg_off" feature.
         """
         assert kernel_size > 0
         assert split_time > 0
@@ -292,7 +329,7 @@ class BackgroundSparseMed(Background):
             # Fill up remainder of index array with last entry
             bg_idx[idx1:] = ii
 
-        self.image_proc.value = self.image_count
+        self.image_proc.value = 1
 
         # Write background data
         pos = 0
@@ -300,8 +337,19 @@ class BackgroundSparseMed(Background):
         while pos < self.image_count:
             stop = min(pos + step, self.image_count)
             cur_slice = slice(pos, stop)
-            self.h5out["events/image_bg"][cur_slice] = \
-                bg_images[bg_idx[cur_slice]]
+            cur_bg_data = bg_images[bg_idx[cur_slice]]
+            self.writer.store_feature_chunk("image_bg", cur_bg_data)
+            if self.offset_correction:
+                # Record background offset correction "bg_off". We take a
+                # slice of 20px from the top of the image (there are normally
+                # no events here, only the channel walls are visible).
+                sh, sw = self.input_data.shape[1:]
+                roi_full = (slice(None), slice(0, 20), slice(0, sw))
+                roi_cur = (cur_slice, slice(0, 20), slice(0, sw))
+                val_bg = np.mean(cur_bg_data[roi_full], axis=(1, 2))
+                val_dat = np.mean(self.input_data[roi_cur], axis=(1, 2))
+                # background image = image_bg + bg_off
+                self.writer.store_feature_chunk("bg_off", val_dat - val_bg)
             pos += step
 
     def process_second(self,
@@ -345,7 +393,7 @@ class BackgroundSparseMed(Background):
 
         self.bg_images[ii] = self.shared_output.reshape(self.image_shape)
 
-        self.image_proc.value = idx_stop
+        self.image_proc.value = idx_stop / self.image_count
 
 
 class WorkerSparseMed(mp_spawn.Process):

dcnum/feat/feat_brightness/bright_all.py

@@ -4,16 +4,38 @@ import numpy as np
 from .common import brightness_names
 
 
-def brightness_features(image,
-                        mask,
-                        image_bg=None,
-                        image_corr=None):
+def brightness_features(image: np.ndarray[np.uint8],
+                        mask: np.ndarray[np.bool_],
+                        image_bg: np.ndarray[np.uint8] = None,
+                        image_corr: np.ndarray[np.int16] = None,
+                        bg_off: float = None,
+                        ):
+    """Compute brightness features
+
+    Parameters
+    ----------
+    image: np.ndarray
+        2D array of "image" of shape (H, W)
+    mask: np.ndarray
+        3D array containing the N masks of shape (N, H, W)
+    image_bg: np.ndarray
+        2D array of "image_bg" of shape (H, W), required for computing
+        the "bg_med" feature.
+    image_corr: np.ndarray
+        2D array of (image - image_bg), which can be optionally passed
+        to this method. If not given, will be computed.
+    bg_off: float
+        Systematic offset value for correcting the brightness of the
+        background data which has an effect on "bright_bc_avg",
+        "bright_perc_10", "bright_perc_90", and "bg_med" (`bg_off` is
+        generated by sparsemed background correction).
+    """
     mask = np.array(mask, dtype=bool)
     size = mask.shape[0]
 
     br_dict = {}
-    for key in brightness_names:
-        br_dict[key] = np.full(size, np.nan, dtype=np.float64)
+    for mkey in brightness_names:
+        br_dict[mkey] = np.full(size, np.nan, dtype=np.float64)
 
     avg_sd = compute_avg_sd_masked_uint8(image, mask)
     br_dict["bright_avg"][:] = avg_sd[:, 0]
@@ -36,6 +58,19 @@ def brightness_features(image,
         br_dict["bright_perc_10"][:] = percentiles[:, 0]
         br_dict["bright_perc_90"][:] = percentiles[:, 1]
 
+    if bg_off is not None:
+        # subtract the background offset for all values that are computed
+        # from background-corrected images
+        for mkey in ["bright_bc_avg", "bright_perc_10", "bright_perc_90"]:
+            if mkey in br_dict:
+                br_dict[mkey] -= bg_off
+
+        # add the background offset to all values that were computed from
+        # the background only
+        for pkey in ["bg_med"]:
+            if pkey in br_dict:
+                br_dict[pkey] += bg_off
+
     return br_dict
 
 

dcnum/feat/feat_contour/__init__.py

@@ -0,0 +1,4 @@
+# flake8: noqa: F401
+"""Feature computation: OpenCV moments-based features"""
+from .moments import moments_based_features
+from .volume import volume_from_contours

dcnum/feat/{feat_moments/mt_legacy.py → feat_contour/moments.py}

@@ -2,11 +2,27 @@ import cv2
 import numpy as np
 
 
-from .ct_opencv import contour_single_opencv
-
-
-def moments_based_features(mask, pixel_size):
+from .contour import contour_single_opencv
+
+
+def moments_based_features(
+        mask: np.ndarray,
+        pixel_size: float,
+        ret_contour: bool = False,
+        ):
+    """Compute moment-based features for a mask image
+
+    Parameters
+    ----------
+    mask: np.ndarray
+        3D stack of 2D boolean mask images to analyze
+    pixel_size: float
+        pixel size of the mask image in µm
+    ret_contour: bool
+        whether to also return the raw contour
+    """
     assert pixel_size is not None and pixel_size != 0
+    raw_contours = []
 
     size = mask.shape[0]
 
@@ -42,9 +58,13 @@ def moments_based_features(mask, pixel_size):
     for ii in range(size):
         # raw contour
         cont_raw = contour_single_opencv(mask[ii])
-        if len(cont_raw.shape) < 2:
-            continue
-        if cv2.contourArea(cont_raw) == 0:
+        # only continue if the contour is valid
+        not_valid = len(cont_raw.shape) < 2 or cv2.contourArea(cont_raw) == 0
+
+        if ret_contour:
+            raw_contours.append(None if not_valid else cont_raw)
+
+        if not_valid:
             continue
 
         mu_raw = cv2.moments(cont_raw)
@@ -53,6 +73,7 @@ def moments_based_features(mask, pixel_size):
 
         # convex hull
         cont_cvx = np.squeeze(cv2.convexHull(cont_raw))
+
         mu_cvx = cv2.moments(cont_cvx)
         arc_cvx = np.float64(cv2.arcLength(cont_cvx, True))
 
@@ -110,7 +131,7 @@ def moments_based_features(mask, pixel_size):
         # specify validity
         valid[ii] = True
 
-    return {
+    data = {
         "area_msd": feat_area_msd,
         "area_ratio": feat_area_ratio,
         "area_um": feat_area_um,
@@ -131,3 +152,6 @@ def moments_based_features(mask, pixel_size):
         "tilt": feat_tilt,
         "valid": valid,
     }
+    if ret_contour:
+        data["contour"] = raw_contours
+    return data