dcnum 0.13.2__py3-none-any.whl → 0.23.1__py3-none-any.whl

dcnum/feat/gate.py

@@ -1,68 +1,64 @@
+"""Feature gating"""
 import copy
+import numbers
+import warnings
 
 import numpy as np
 
-from ..meta.ppid import kwargs_to_ppid
+from ..meta.ppid import kwargs_to_ppid, ppid_to_kwargs
 
 
 class Gate:
     #: the default value for `size_thresh_mask` if not given as kwarg
     _default_size_thresh_mask = 10
 
-    def __init__(self, data, *, online_gates: bool = False,
+    def __init__(self, data, *,
+                 online_gates: bool = False,
                  size_thresh_mask: int = None):
         """Gate feature data
 
         Parameters
         ----------
         data: .HDF5Data
-            dcevent data instance
+            dcnum data instance
         online_gates: bool
-            set to True to enable gating with `online_filters`
+            set to True to enable gating with "online" gates stored
+            in the input file; online gates are applied in real-time
+            deformability cytometry before writing data to disk during
+            a measurement
         size_thresh_mask: int
             Only masks with more pixels than `size_thresh_mask` are
-            considered to be a valid event; Originally, this
-            `trig_thresh` value defaulted to 200, but this seemed to
-            be a little too large, defaults to 10.
+            considered to be a valid event; Originally, the
+            `bin area min`/`trig_thresh` value defaulted to 200 which is
+            too large; defaults to 10 or the original value in case
+            `online_gates` is set.
         """
-        #: dcevent .HDF5Data instance
-        self.data = data
-
-        #: gating keyword arguments
-        self.kwargs = {}
+        #: box gating (value range for each feature)
+        self.box_gates = {}
 
         if online_gates:
-            self.box_gates = self._compute_online_gates()
-            # Set triggering threshold to value from source dataset
-            self._set_kwarg("size_thresh_mask", "online_contour",
-                            "bin area min", size_thresh_mask)
-            # If the user did not provide a value and there is nothing in the
-            # original file, then set the default value.
-            if self.kwargs.get("size_thresh_mask") is None:
-                self.kwargs["size_thresh_mask"] = \
-                    self._default_size_thresh_mask
-        else:
-            self.box_gates = {}
-            # If the user did not provide a size_thresh_mask, use the default.
+            # Deal with online gates.
+            # First, compute the box gates.
+            self.box_gates.update(self._extract_online_gates(data))
+            # If the user did not specify a threshold, attempt to extract
+            # it from the metadata.
             if size_thresh_mask is None:
-                size_thresh_mask = self._default_size_thresh_mask
-            self.kwargs["size_thresh_mask"] = size_thresh_mask
-
-        self.kwargs["online_gates"] = online_gates
-
-    def _set_kwarg(self, name, sec, key, user_value):
-        if user_value is None:
-            value = self.data.meta_nest.get(sec, {}).get(key)
-        else:
-            value = user_value
-        if value is not None:
-            self.kwargs[name] = value
+                size_thresh_mask = data.meta_nest.get(
+                    "online_contour", {}).get("bin area min")
 
-    def _compute_online_gates(self):
-        all_online_filters = {}
+        #: gating keyword arguments
+        self.kwargs = {
+            "online_gates": online_gates,
+            # Set the size threshold, defaulting to `_default_size_thresh_mask`
+            "size_thresh_mask":
+                size_thresh_mask or self._default_size_thresh_mask
+        }
+
+    def _extract_online_gates(self, data):
+        ogates = {}
         # Extract online filters from the dataset
-        of = self.data.meta_nest.get("online_filter", {})
-        for key in of:
+        source_meta = data.meta_nest.get("online_filter", {})
+        for key in source_meta:
             if key.endswith("polygon points"):
                 raise NotImplementedError("Polygon gating not implemented!")
             elif (key.endswith("soft limit")
@@ -70,25 +66,36 @@ class Gate:
                 # we only want hard gates
                 continue
             else:
-                # only add the filter if it is not a soft limit
-                sl = of.get(f"{key.rsplit(' ', 1)[0]} soft limit", True)
-                if not sl:
-                    all_online_filters[key] = of[key]
+                try:
+                    feat, lim = key.rsplit(' ', 1)
+                    lim_idx = ["min", "max"].index(lim)
+                except ValueError:
+                    warnings.warn(f"Unexpected online gate '{key}'")
+                else:
+                    # make sure we are not dealing with a soft limit
+                    if not source_meta.get(f"{feat} soft limit", True):
+                        ogates.setdefault(feat, [None, None])
+                        ogates[feat][lim_idx] = source_meta[key]
 
         # This is somehow hard-coded in Shape-In (minimum size is 3px)
-        px_size = self.data.pixel_size
-        all_online_filters["size_x min"] = max(
-            all_online_filters.get("size_x min", 0), 3 * px_size)
-        all_online_filters["size_y min"] = max(
-            all_online_filters.get("size_y min", 0), 3 * px_size)
+        px_size = data.pixel_size
+        ogates["size_x"] = [
+            max(ogates.get("size_x min", 0), 3 * px_size), None]
+        ogates["size_y"] = [
+            max(ogates.get("size_y min", 0), 3 * px_size), None]
 
-        return all_online_filters
+        return ogates
+
+    @property
+    def features(self):
+        """Sorted list of feature gates defined"""
+        return sorted(self.box_gates.keys())
 
     def get_ppid(self):
         """Return a unique gating pipeline identifier
 
         The pipeline identifier is universally applicable and must
-        be backwards-compatible (future versions of dcevent will
+        be backwards-compatible (future versions of dcnum will
         correctly acknowledge the ID).
 
         The gating pipeline ID is defined as::
@@ -100,10 +107,15 @@ class Gate:
 
             online_gates=True^size_thresh_mask=5
         """
-        return self.get_ppid_from_kwargs(self.kwargs)
+        return self.get_ppid_from_ppkw(self.kwargs)
+
+    @classmethod
+    def get_ppid_code(cls):
+        return "norm"
 
     @classmethod
-    def get_ppid_from_kwargs(cls, kwargs):
+    def get_ppid_from_ppkw(cls, kwargs):
+        """return full pipeline identifier from the given keywords"""
         # TODO:
         #  If polygon filters are used, the MD5sum should be used and
         #  they should be placed as a log to the output .rtdc file.
@@ -111,27 +123,21 @@ class Gate:
         if kwargs.get("size_thresh_mask") is None:
             # Set the default described in init
             kwargs["size_thresh_mask"] = cls._default_size_thresh_mask
-        key = cls.key()
+        key = cls.get_ppid_code()
         cback = kwargs_to_ppid(cls, "__init__", kwargs)
 
         return ":".join([key, cback])
 
-    @property
-    def features(self):
-        return [kk.split()[0] for kk in list(self.box_gates.keys())]
-
-    def gate_feature(self, feat, data):
-        valid_left = True
-        valid_right = True
-        if f"{feat} min" in self.box_gates:
-            valid_left = data > self.box_gates[f"{feat} min"]
-        if f"{feat} max" in self.box_gates:
-            valid_right = data < self.box_gates[f"{feat} max"]
-        return np.logical_and(valid_left, valid_right)
-
-    @classmethod
-    def key(cls):
-        return "norm"
+    @staticmethod
+    def get_ppkw_from_ppid(gate_ppid):
+        code, pp_gate_kwargs = gate_ppid.split(":")
+        if code != Gate.get_ppid_code():
+            raise ValueError(
+                f"Could not find gating method '{code}'!")
+        kwargs = ppid_to_kwargs(cls=Gate,
+                                method="__init__",
+                                ppid=pp_gate_kwargs)
+        return kwargs
 
     def gate_event(self, event):
         """Return None if the event should not be used, else `event`"""
@@ -157,6 +163,22 @@ class Gate:
             raise ValueError("Empty events provided!")
         return valid
 
+    def gate_feature(self,
+                     feat: str,
+                     data: numbers.Number | np.ndarray):
+        """Return boolean indicating whether `data` value is in box gate
+
+        `data` may be a number or an array. If no box filter is defined
+        for `feat`, `True` is always returned. Otherwise, either a boolean
+        or a boolean array is returned, depending on the type of `data`.
+        Not that `np.logical_and` can deal with mixed argument types
+        (scalar and array).
+        """
+        bound_lo, bound_up = self.box_gates[feat]
+        valid_lo = data >= bound_lo if bound_lo is not None else True
+        valid_up = data <= bound_up if bound_up is not None else True
+        return np.logical_and(valid_lo, valid_up)
+
     def gate_mask(self, mask, mask_sum=None):
         """Gate the mask, return False if the mask should not be used
 

dcnum/feat/queue_event_extractor.py

@@ -1,3 +1,4 @@
+"""Feature Extraction: event extractor worker"""
 import collections
 import logging
 from logging.handlers import QueueHandler
@@ -9,11 +10,11 @@ import traceback
 
 import numpy as np
 
-from ..meta.ppid import kwargs_to_ppid
+from ..meta.ppid import kwargs_to_ppid, ppid_to_kwargs
 from ..read import HDF5Data
 
 from .feat_brightness import brightness_features
-from .feat_moments import moments_based_features
+from .feat_contour import moments_based_features, volume_from_contours
 from .feat_texture import haralick_texture_features
 from .gate import Gate
 
@@ -27,15 +28,17 @@ class QueueEventExtractor:
     def __init__(self,
                  data: HDF5Data,
                  gate: Gate,
-                 preselect: bool,
-                 ptp_median: float,
                  raw_queue: mp.Queue,
                  event_queue: mp.Queue,
                  log_queue: mp.Queue,
                  feat_nevents: mp.Array,
                  label_array: mp.Array,
                  finalize_extraction: mp.Value,
+                 invalid_mask_counter: mp.Value,
+                 worker_monitor: mp.RawArray,
+                 log_level: int = None,
                  extract_kwargs: dict = None,
+                 worker_index: int = None,
                  *args, **kwargs):
         """Base class for event extraction from label images
 
@@ -44,15 +47,10 @@ class QueueEventExtractor:
 
         Parameters
         ----------
-        data:
+        data: HDF5Data
             Data source.
-        gate:
+        gate: Gate
             Gating rules.
-        preselect:
-            Whether to perform data preselection based on peak-to-peak
-            values in the images.
-        ptp_median:
-            Median peak-to-peak value in the images for preselction.
         raw_queue:
             Queue from which the worker obtains the chunks and
             indices of the labels to work on.
@@ -70,27 +68,42 @@ class QueueEventExtractor:
         finalize_extraction:
             Shared value indicating whether this worker should stop as
             soon as the `raw_queue` is empty.
+        invalid_mask_counter:
+            Counts masks labeled as invalid by the feature extractor
+        worker_monitor:
+            Monitors the frames each worker has processed. Only the
+            value in `worker_monitor[worker_index]` is modified.
+        log_level:
+            Logging level to use
         extract_kwargs:
             Keyword arguments for the extraction process. See the
             keyword-only arguments in
             :func:`QueueEventExtractor.get_events_from_masks`.
-
+        worker_index:
+            The index to increment values in `worker_monitor`
         """
         super(QueueEventExtractor, self).__init__(*args, **kwargs)
+        #: Worker index for populating
+        self.worker_index = worker_index or 0
         #: Data instance
         self.data = data
         #: Gating information
         self.gate = gate
-        #: Whether to perform Preselection
-        self.preselect = preselect
-        #: Peak-to-peak median for preselection
-        self.ptp_median = ptp_median
         #: queue containing sub-indices for `label_array`
         self.raw_queue = raw_queue
         #: queue with event-wise feature dictionaries
         self.event_queue = event_queue
         #: queue for logging
         self.log_queue = log_queue
+        #: invalid mask counter
+        self.invalid_mask_counter = invalid_mask_counter
+        #: worker busy counter
+        self.worker_monitor = worker_monitor
+        # Logging needs to be set up after `start` is called, otherwise
+        # it looks like we have the same PID as the parent process. We
+        # are setting up logging in `run`.
+        self.logger = None
+        self.log_level = log_level or logging.getLogger("dcnum").level
         #: Shared array of length `len(data)` into which the number of
         #: events per frame is written.
         self.feat_nevents = feat_nevents
@@ -105,30 +118,48 @@ class QueueEventExtractor:
         extract_kwargs.setdefault("haralick", True)
         #: Feature extraction keyword arguments.
         self.extract_kwargs = extract_kwargs
-        # Logging needs to be set up after `start` is called, otherwise
-        # it looks like we have the same PID as the parent process. We
-        # are setting up logging in `run`.
-        self.logger = None
 
     @staticmethod
-    def get_init_kwargs(data, gate, preselect, ptp_median, log_queue):
-        """You can pass `*args.values()` directly to __init__
+    def get_init_kwargs(data: HDF5Data,
+                        gate: Gate,
+                        num_extractors: int,
+                        log_queue: mp.Queue,
+                        log_level: int = None,
+                        ):
+        """Get initialization arguments for :cass:`.QueueEventExtractor`
 
         This method was created for convenience reasons:
         - It makes sure that the order of arguments is correct, since it
           is implemented in the same class.
         - It simplifies testing.
+
+        Parameters
+        ----------
+        data: HDF5Data
+            Input data
+        gate: HDF5Data
+            Gating class to use
+        num_extractors: int
+            Number of extractors that will be used
+        log_queue: mp.Queue
+            Queue the worker uses for sending log messages
+        log_level: int
+            Logging level to use in the worker process
+
+        Returns
+        -------
+        args: dict
+            You can pass `*args.values()` directly to `__init__`
         """
         # queue with the raw (unsegmented) image data
         raw_queue = mp_spawn.Queue()
         # queue with event-wise feature dictionaries
         event_queue = mp_spawn.Queue()
 
+        # Note that the order must be identical to  __init__
         args = collections.OrderedDict()
         args["data"] = data
         args["gate"] = gate
-        args["preselect"] = preselect
-        args["ptp_median"] = ptp_median
         args["raw_queue"] = raw_queue
         args["event_queue"] = event_queue
         args["log_queue"] = log_queue
@@ -139,37 +170,54 @@ class QueueEventExtractor:
             np.ctypeslib.ctypes.c_int16,
             int(np.prod(data.image.chunk_shape)))
         args["finalize_extraction"] = mp_spawn.Value("b", False)
+        args["invalid_mask_counter"] = mp_spawn.Value("L", 0)
+        args["worker_monitor"] = mp_spawn.RawArray("L", num_extractors)
+        args["log_level"] = log_level or logging.getLogger("dcnum").level
         return args
 
-    @classmethod
-    def get_ppid_from_kwargs(cls, kwargs):
-        """Return the pipeline ID for this event extractor"""
-        key = "legacy"
-        cback = kwargs_to_ppid(cls, "get_events_from_masks", kwargs)
-        return ":".join([key, cback])
-
     def get_events_from_masks(self, masks, data_index, *,
                               brightness: bool = True,
                               haralick: bool = True,
+                              volume: bool = True,
                               ):
         """Get events dictionary, performing event-based gating"""
         events = {"mask": masks}
         image = self.data.image[data_index][np.newaxis]
         image_bg = self.data.image_bg[data_index][np.newaxis]
         image_corr = self.data.image_corr[data_index][np.newaxis]
+        if "bg_off" in self.data:
+            bg_off = self.data["bg_off"][data_index]
+        else:
+            bg_off = None
 
         events.update(
             moments_based_features(
                 masks,
-                pixel_size=self.data.pixel_size))
+                pixel_size=self.data.pixel_size,
+                ret_contour=volume,
+                ))
+
         if brightness:
             events.update(brightness_features(
-                mask=masks, image=image, image_bg=image_bg,
+                mask=masks,
+                image=image,
+                image_bg=image_bg,
+                bg_off=bg_off,
                 image_corr=image_corr
             ))
         if haralick:
             events.update(haralick_texture_features(
-                mask=masks, image=image, image_corr=image_corr
+                mask=masks,
+                image=image,
+                image_corr=image_corr,
+            ))
+
+        if volume:
+            events.update(volume_from_contours(
+                contour=events.pop("contour"),  # remove contour from events!
+                pos_x=events["pos_x"],
+                pos_y=events["pos_y"],
+                pixel_size=self.data.pixel_size,
             ))
 
         # gating on feature arrays
@@ -193,8 +241,7 @@ class QueueEventExtractor:
         # over from gated_events to valid_events. According to our experience
         # invalid events happen rarely though.
         if np.any(invalid):
-            self.logger.info(f"Discarded {np.sum(invalid)} events due to "
-                             "invalid segmentation.")
+            self.invalid_mask_counter.value += np.sum(invalid)
             for key in gated_events:
                 valid_events[key] = gated_events[key][valid]
         else:
@@ -219,7 +266,7 @@ class QueueEventExtractor:
         """Return a unique feature extractor pipeline identifier
 
         The pipeline identifier is universally applicable and must
-        be backwards-compatible (future versions of dcevent will
+        be backwards-compatible (future versions of dcnum will
         correctly acknowledge the ID).
 
         The feature extractor pipeline ID is defined as::
@@ -236,7 +283,29 @@ class QueueEventExtractor:
 
             b=1^h=1
         """
-        return self.get_ppid_from_kwargs(self.extract_kwargs)
+        return self.get_ppid_from_ppkw(self.extract_kwargs)
+
+    @classmethod
+    def get_ppid_code(cls):
+        return "legacy"
+
+    @classmethod
+    def get_ppid_from_ppkw(cls, kwargs):
+        """Return the pipeline ID for this event extractor"""
+        code = cls.get_ppid_code()
+        cback = kwargs_to_ppid(cls, "get_events_from_masks", kwargs)
+        return ":".join([code, cback])
+
+    @staticmethod
+    def get_ppkw_from_ppid(extr_ppid):
+        code, pp_extr_kwargs = extr_ppid.split(":")
+        if code != QueueEventExtractor.get_ppid_code():
+            raise ValueError(
+                f"Could not find extraction method '{code}'!")
+        kwargs = ppid_to_kwargs(cls=QueueEventExtractor,
+                                method="get_events_from_masks",
+                                ppid=pp_extr_kwargs)
+        return kwargs
 
     def process_label(self, label, index):
         """Process one label image, extracting masks and features"""
@@ -245,11 +314,7 @@ class QueueEventExtractor:
             # TODO: Do this before segmentation already?
             # skip events that have been analyzed already
             return None
-        if self.preselect:
-            # TODO: Do this before segmentation already?
-            ptp = np.ptp(self.data.image_corr[index])
-            if ptp < 0.1 * self.ptp_median:
-                return None
+
         masks = self.get_masks_from_label(label)
         if masks.size:
             events = self.get_events_from_masks(
@@ -260,19 +325,30 @@ class QueueEventExtractor:
 
     def run(self):
         """Main loop of worker process"""
+        self.worker_monitor[self.worker_index] = 0
         # Don't wait for these two queues when joining workers
         self.raw_queue.cancel_join_thread()
-        self.log_queue.cancel_join_thread()
         #: logger sends all logs to `self.log_queue`
         self.logger = logging.getLogger(
             f"dcnum.feat.EventExtractor.{os.getpid()}")
+        self.logger.setLevel(self.log_level)
+        # Clear any handlers that might be set for this logger. This is
+        # important for the case when we are an instance of
+        # EventExtractorThread, because then all handlers from the main
+        # thread are inherited (as opposed to no handlers in the case
+        # of EventExtractorProcess).
+        self.logger.handlers.clear()
         queue_handler = QueueHandler(self.log_queue)
+        queue_handler.setLevel(self.log_level)
         self.logger.addHandler(queue_handler)
-        self.logger.addFilter(DeduplicatingLoggingFilter())
-        self.logger.debug(f"Running {self} in PID {os.getpid()}")
+        self.logger.info("Ready")
 
         mp_array = np.ctypeslib.as_array(
             self.label_array).reshape(self.data.image.chunk_shape)
+
+        # only close queues when we have created them ourselves.
+        close_queues = isinstance(self, EventExtractorProcess)
+
         while True:
             try:
                 chunk_index, label_index = self.raw_queue.get(timeout=.03)
@@ -281,8 +357,7 @@ class QueueEventExtractor:
                 if self.finalize_extraction.value:
                     # The manager told us that there is nothing more coming.
                     self.logger.debug(
-                        f"Finalizing worker {self} with PID {os.getpid()}. "
-                        f"{self.event_queue.qsize()} events are still queued.")
+                        f"Finalizing worker {self} with PID {os.getpid()}")
                     break
             else:
                 try:
@@ -298,16 +373,24 @@ class QueueEventExtractor:
                     else:
                         self.feat_nevents[index] = 0
                     self.event_queue.put((index, events))
+                self.worker_monitor[self.worker_index] += 1
 
         self.logger.debug(f"Finalizing `run` for PID {os.getpid()}, {self}")
-        # Explicitly close the event queue and join it
-        self.event_queue.close()
-        self.event_queue.join_thread()
-        self.logger.debug(f"End of `run` for PID {os.getpid()}, {self}")
-        # Also close the logging queue. Not that not all messages might
-        # arrive in the logging queue, since we called `cancel_join_thread`
-        # earlier.
-        self.log_queue.close()
+        if close_queues:
+            # Explicitly close the event queue and join it
+            self.event_queue.close()
+            self.event_queue.join_thread()
+            self.logger.debug(f"End of `run` for PID {os.getpid()}, {self}")
+
+        # Make sure everything gets written to the queue.
+        queue_handler.flush()
+
+        if close_queues:
+            # Also close the logging queue. Note that not all messages might
+            # arrive in the logging queue, since we called `cancel_join_thread`
+            # earlier.
+            self.log_queue.close()
+            self.log_queue.join_thread()
 
 
 class EventExtractorProcess(QueueEventExtractor, mp_spawn.Process):
@@ -322,17 +405,3 @@ class EventExtractorThread(QueueEventExtractor, threading.Thread):
     def __init__(self, *args, **kwargs):
         super(EventExtractorThread, self).__init__(
             name="EventExtractorThread", *args, **kwargs)
-
-
-class DeduplicatingLoggingFilter(logging.Filter):
-    def __init__(self, *args, **kwargs):
-        super(DeduplicatingLoggingFilter, self).__init__(*args, **kwargs)
-        self._records = []
-
-    def filter(self, record):
-        """Return True if the record should be logged"""
-        msg = record.getMessage()
-        logged = msg in self._records
-        if not logged:
-            self._records.append(msg)
-        return not logged

dcnum/logic/__init__.py

@@ -0,0 +1,5 @@
+# flake8: noqa: F401
+"""Logic for running the dcnum pipeline"""
+from .ctrl import DCNumJobRunner
+from .job import DCNumPipelineJob
+from .json_encoder import ExtendedJSONEncoder