dcnum 0.14.0__py3-none-any.whl → 0.15.0__py3-none-any.whl

dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.14.0'
-__version_tuple__ = version_tuple = (0, 14, 0)
+__version__ = version = '0.15.0'
+__version_tuple__ = version_tuple = (0, 15, 0)

dcnum/feat/__init__.py

@@ -1,4 +1,5 @@
 # flake8: noqa: F401
+"""Feature computation"""
 from . import feat_background, feat_brightness, feat_moments, feat_texture
 from .event_extractor_manager_thread import EventExtractorManagerThread
 from .queue_event_extractor import (

dcnum/feat/event_extractor_manager_thread.py

@@ -1,3 +1,4 @@
+"""Feature computation: managing event extraction threads"""
 import logging
 import multiprocessing as mp
 import threading
@@ -45,6 +46,8 @@ class EventExtractorManagerThread(threading.Thread):
         """
         super(EventExtractorManagerThread, self).__init__(
               name="EventExtractorManager", *args, **kwargs)
+        if debug:
+            fe_kwargs["close_queues"] = False
         self.logger = logging.getLogger(
             "dcnum.feat.EventExtractorManagerThread")
         #: Keyword arguments for class:`.EventExtractor`

dcnum/feat/feat_background/__init__.py

@@ -1,16 +1,6 @@
 # flake8: noqa: F401
-import functools
-
-from .base import Background
+"""Feature computation: background image data from image data"""
+from .base import Background, get_available_background_methods
 # Background methods are registered by importing them here.
 from .bg_roll_median import BackgroundRollMed
 from .bg_sparse_median import BackgroundSparseMed
-
-
-@functools.cache
-def get_available_background_methods():
-    """Return dictionary of background computation methods"""
-    methods = {}
-    for cls in Background.__subclasses__():
-        methods[cls.key()] = cls
-    return methods

dcnum/feat/feat_background/base.py

@@ -1,8 +1,10 @@
 import abc
+import functools
 import inspect
 import multiprocessing as mp
 import pathlib
 import uuid
+import warnings
 
 import h5py
 import hdf5plugin
@@ -131,37 +133,6 @@ class Background(abc.ABC):
         if self.h5in is not self.h5out and self.h5out is not None:
             self.h5out.close()
 
-    @staticmethod
-    def get_kwargs_from_ppid(bg_ppid):
-        """Return keyword arguments for any subclass from a PPID string"""
-        name, pp_check_user_kwargs = bg_ppid.split(":")
-        for cls in Background.__subclasses__():
-            if cls.key() == name:
-                break
-        else:
-            raise ValueError(
-                f"Could not find background computation method '{name}'!")
-        kwargs = ppid.ppid_to_kwargs(cls=cls,
-                                     method="check_user_kwargs",
-                                     ppid=pp_check_user_kwargs)
-        return kwargs
-
-    @classmethod
-    def get_ppid_from_kwargs(cls, kwargs):
-        """Return the PPID based on given keyword arguments for a subclass"""
-        key = cls.key()
-        cback = ppid.kwargs_to_ppid(cls, "check_user_kwargs", kwargs)
-        return ":".join([key, cback])
-
-    @classmethod
-    def key(cls):
-        if cls is Background:
-            raise ValueError("Cannot get `key` for `Background` base class!")
-        key = cls.__name__.lower()
-        if key.startswith("background"):
-            key = key[10:]
-        return key
-
     @abc.abstractmethod
     def check_user_kwargs(self, **kwargs):
         """Implement this to check the kwargs during init"""
@@ -186,11 +157,42 @@ class Background(abc.ABC):
 
             k=100^b=10000
         """
-        return self.get_ppid_from_kwargs(self.kwargs)
+        return self.get_ppid_from_ppkw(self.kwargs)
+
+    @classmethod
+    def get_ppid_code(cls):
+        if cls is Background:
+            raise ValueError("Cannot get `key` for `Background` base class!")
+        key = cls.__name__.lower()
+        if key.startswith("background"):
+            key = key[10:]
+        return key
+
+    @classmethod
+    def get_ppid_from_ppkw(cls, kwargs):
+        """Return the PPID based on given keyword arguments for a subclass"""
+        code = cls.get_ppid_code()
+        cback = ppid.kwargs_to_ppid(cls, "check_user_kwargs", kwargs)
+        return ":".join([code, cback])
+
+    @staticmethod
+    def get_ppkw_from_ppid(bg_ppid):
+        """Return keyword arguments for any subclass from a PPID string"""
+        code, pp_check_user_kwargs = bg_ppid.split(":")
+        for bg_code in get_available_background_methods():
+            if bg_code == code:
+                cls = get_available_background_methods()[bg_code]
+                break
+        else:
+            raise ValueError(
+                f"Could not find background computation method '{code}'!")
+        kwargs = ppid.ppid_to_kwargs(cls=cls,
+                                     method="check_user_kwargs",
+                                     ppid=pp_check_user_kwargs)
+        return kwargs
 
     def process(self):
         self.process_approach()
-
         bg_ppid = self.get_ppid()
         # Store pipeline information in the image_bg feature
         self.h5out["events/image_bg"].attrs["dcnum ppid background"] = bg_ppid
@@ -200,3 +202,19 @@ class Background(abc.ABC):
     @abc.abstractmethod
     def process_approach(self):
         """The actual background computation approach"""
+
+    @classmethod
+    def get_ppid_from_kwargs(cls, kwargs):
+        warnings.warn(
+            "Please use get_ppid_from_ppkw instead of get_ppid_from_kwargs.",
+            DeprecationWarning)
+        return cls.get_ppid_from_ppkw(kwargs)
+
+
+@functools.cache
+def get_available_background_methods():
+    """Return dictionary of background computation methods"""
+    methods = {}
+    for cls in Background.__subclasses__():
+        methods[cls.get_ppid_code()] = cls
+    return methods

dcnum/feat/feat_brightness/__init__.py

@@ -1,3 +1,4 @@
 # flake8: noqa: F401
+"""Feature computation: brightness-based features"""
 from .bright_all import brightness_features
 from .common import brightness_names

dcnum/feat/feat_moments/__init__.py

@@ -1,3 +1,4 @@
 # flake8: noqa: F401
+"""Feature computation: OpenCV moments-based features"""
 from .mt_legacy import moments_based_features
 

dcnum/feat/feat_texture/__init__.py

@@ -1,2 +1,3 @@
 # flake8: noqa: F401
+"""Feature computation: Haralick texture features"""
 from .tex_all import haralick_names, haralick_texture_features

dcnum/feat/gate.py

@@ -1,8 +1,10 @@
+"""Feature computation: gating after feature extraction"""
 import copy
+import warnings
 
 import numpy as np
 
-from ..meta.ppid import kwargs_to_ppid
+from ..meta.ppid import kwargs_to_ppid, ppid_to_kwargs
 
 
 class Gate:
@@ -84,42 +86,6 @@ class Gate:
 
         return all_online_filters
 
-    def get_ppid(self):
-        """Return a unique gating pipeline identifier
-
-        The pipeline identifier is universally applicable and must
-        be backwards-compatible (future versions of dcevent will
-        correctly acknowledge the ID).
-
-        The gating pipeline ID is defined as::
-
-            KEY:KW_GATE
-
-        Where KEY is e.g. "online_gates", and KW_GATE is
-        the corresponding value, e.g.::
-
-            online_gates=True^size_thresh_mask=5
-        """
-        return self.get_ppid_from_kwargs(self.kwargs)
-
-    @classmethod
-    def get_ppid_from_kwargs(cls, kwargs):
-        # TODO:
-        #  If polygon filters are used, the MD5sum should be used and
-        #  they should be placed as a log to the output .rtdc file.
-        kwargs = copy.deepcopy(kwargs)
-        if kwargs.get("size_thresh_mask") is None:
-            # Set the default described in init
-            kwargs["size_thresh_mask"] = cls._default_size_thresh_mask
-        key = cls.key()
-        cback = kwargs_to_ppid(cls, "__init__", kwargs)
-
-        return ":".join([key, cback])
-
-    @property
-    def features(self):
-        return [kk.split()[0] for kk in list(self.box_gates.keys())]
-
     def gate_feature(self, feat, data):
         valid_left = True
         valid_right = True
@@ -129,10 +95,6 @@ class Gate:
             valid_right = data < self.box_gates[f"{feat} max"]
         return np.logical_and(valid_left, valid_right)
 
-    @classmethod
-    def key(cls):
-        return "norm"
-
     def gate_event(self, event):
         """Return None if the event should not be used, else `event`"""
         if self.box_gates and event:
@@ -170,3 +132,62 @@ class Gate:
         if mask_sum is None:
             mask_sum = np.sum(mask)
         return mask_sum > self.kwargs["size_thresh_mask"]
+
+    def get_ppid(self):
+        """Return a unique gating pipeline identifier
+
+        The pipeline identifier is universally applicable and must
+        be backwards-compatible (future versions of dcevent will
+        correctly acknowledge the ID).
+
+        The gating pipeline ID is defined as::
+
+            KEY:KW_GATE
+
+        Where KEY is e.g. "online_gates", and KW_GATE is
+        the corresponding value, e.g.::
+
+            online_gates=True^size_thresh_mask=5
+        """
+        return self.get_ppid_from_ppkw(self.kwargs)
+
+    @classmethod
+    def get_ppid_code(cls):
+        return "norm"
+
+    @classmethod
+    def get_ppid_from_ppkw(cls, kwargs):
+        """return full pipeline identifier from the given keywords"""
+        # TODO:
+        #  If polygon filters are used, the MD5sum should be used and
+        #  they should be placed as a log to the output .rtdc file.
+        kwargs = copy.deepcopy(kwargs)
+        if kwargs.get("size_thresh_mask") is None:
+            # Set the default described in init
+            kwargs["size_thresh_mask"] = cls._default_size_thresh_mask
+        key = cls.get_ppid_code()
+        cback = kwargs_to_ppid(cls, "__init__", kwargs)
+
+        return ":".join([key, cback])
+
+    @staticmethod
+    def get_ppkw_from_ppid(gate_ppid):
+        code, pp_gate_kwargs = gate_ppid.split(":")
+        if code != Gate.get_ppid_code():
+            raise ValueError(
+                f"Could not find gating method '{code}'!")
+        kwargs = ppid_to_kwargs(cls=Gate,
+                                method="__init__",
+                                ppid=pp_gate_kwargs)
+        return kwargs
+
+    @property
+    def features(self):
+        return [kk.split()[0] for kk in list(self.box_gates.keys())]
+
+    @classmethod
+    def get_ppid_from_kwargs(cls, kwargs):
+        warnings.warn(
+            "Please use get_ppid_from_ppkw instead of get_ppid_from_kwargs.",
+            DeprecationWarning)
+        return cls.get_ppid_from_ppkw(kwargs)

dcnum/feat/queue_event_extractor.py

@@ -1,3 +1,4 @@
+"""Feature Extraction: event extractor worker"""
 import collections
 import logging
 from logging.handlers import QueueHandler
@@ -6,10 +7,11 @@ import os
 import queue
 import threading
 import traceback
+import warnings
 
 import numpy as np
 
-from ..meta.ppid import kwargs_to_ppid
+from ..meta.ppid import kwargs_to_ppid, ppid_to_kwargs
 from ..read import HDF5Data
 
 from .feat_brightness import brightness_features
@@ -27,14 +29,13 @@ class QueueEventExtractor:
     def __init__(self,
                  data: HDF5Data,
                  gate: Gate,
-                 preselect: bool,
-                 ptp_median: float,
                  raw_queue: mp.Queue,
                  event_queue: mp.Queue,
                  log_queue: mp.Queue,
                  feat_nevents: mp.Array,
                  label_array: mp.Array,
                  finalize_extraction: mp.Value,
+                 close_queues: bool = True,
                  extract_kwargs: dict = None,
                  *args, **kwargs):
         """Base class for event extraction from label images
@@ -44,15 +45,10 @@ class QueueEventExtractor:
 
         Parameters
         ----------
-        data:
+        data: HDF5Data
             Data source.
-        gate:
+        gate: Gate
             Gating rules.
-        preselect:
-            Whether to perform data preselection based on peak-to-peak
-            values in the images.
-        ptp_median:
-            Median peak-to-peak value in the images for preselction.
         raw_queue:
             Queue from which the worker obtains the chunks and
             indices of the labels to work on.
@@ -70,27 +66,26 @@ class QueueEventExtractor:
         finalize_extraction:
             Shared value indicating whether this worker should stop as
             soon as the `raw_queue` is empty.
+        close_queues: bool
+            Whether to close event and logging queues
+            (set to False in debug mode)
         extract_kwargs:
             Keyword arguments for the extraction process. See the
             keyword-only arguments in
             :func:`QueueEventExtractor.get_events_from_masks`.
-
         """
         super(QueueEventExtractor, self).__init__(*args, **kwargs)
         #: Data instance
         self.data = data
         #: Gating information
         self.gate = gate
-        #: Whether to perform Preselection
-        self.preselect = preselect
-        #: Peak-to-peak median for preselection
-        self.ptp_median = ptp_median
         #: queue containing sub-indices for `label_array`
         self.raw_queue = raw_queue
         #: queue with event-wise feature dictionaries
         self.event_queue = event_queue
         #: queue for logging
         self.log_queue = log_queue
+        self.close_queues = close_queues
         #: Shared array of length `len(data)` into which the number of
         #: events per frame is written.
         self.feat_nevents = feat_nevents
@@ -111,24 +106,49 @@ class QueueEventExtractor:
         self.logger = None
 
     @staticmethod
-    def get_init_kwargs(data, gate, preselect, ptp_median, log_queue):
-        """You can pass `*args.values()` directly to __init__
+    def get_init_kwargs(data: HDF5Data,
+                        gate: Gate,
+                        log_queue: mp.Queue,
+                        preselect: None = None,
+                        ptp_median: None = None):
+        """Get initialization arguments for :cass:`.QueueEventExtractor`
 
         This method was created for convenience reasons:
         - It makes sure that the order of arguments is correct, since it
           is implemented in the same class.
         - It simplifies testing.
+
+        Parameters
+        ----------
+        data: HDF5Data
+            Input data
+        gate: HDF5Data
+            Gating class to use
+        log_queue: mp.Queue
+            Queue for sending log messages
+        preselect, ptp_median:
+            Deprecated
+
+        Returns
+        -------
+        args: dict
+            You can pass `*args.values()` directly to `__init__`
         """
         # queue with the raw (unsegmented) image data
         raw_queue = mp_spawn.Queue()
         # queue with event-wise feature dictionaries
         event_queue = mp_spawn.Queue()
 
+        if preselect is not None:
+            warnings.warn("The `preselect` argument is deprecated!",
+                          DeprecationWarning)
+        if ptp_median is not None:
+            warnings.warn("The `ptp_median` argument is deprecated!",
+                          DeprecationWarning)
+
         args = collections.OrderedDict()
         args["data"] = data
         args["gate"] = gate
-        args["preselect"] = preselect
-        args["ptp_median"] = ptp_median
         args["raw_queue"] = raw_queue
         args["event_queue"] = event_queue
         args["log_queue"] = log_queue
@@ -139,15 +159,9 @@ class QueueEventExtractor:
             np.ctypeslib.ctypes.c_int16,
             int(np.prod(data.image.chunk_shape)))
         args["finalize_extraction"] = mp_spawn.Value("b", False)
+        args["close_queues"] = True
         return args
 
-    @classmethod
-    def get_ppid_from_kwargs(cls, kwargs):
-        """Return the pipeline ID for this event extractor"""
-        key = "legacy"
-        cback = kwargs_to_ppid(cls, "get_events_from_masks", kwargs)
-        return ":".join([key, cback])
-
     def get_events_from_masks(self, masks, data_index, *,
                               brightness: bool = True,
                               haralick: bool = True,
@@ -236,7 +250,29 @@ class QueueEventExtractor:
 
             b=1^h=1
         """
-        return self.get_ppid_from_kwargs(self.extract_kwargs)
+        return self.get_ppid_from_ppkw(self.extract_kwargs)
+
+    @classmethod
+    def get_ppid_code(cls):
+        return "legacy"
+
+    @classmethod
+    def get_ppid_from_ppkw(cls, kwargs):
+        """Return the pipeline ID for this event extractor"""
+        code = cls.get_ppid_code()
+        cback = kwargs_to_ppid(cls, "get_events_from_masks", kwargs)
+        return ":".join([code, cback])
+
+    @staticmethod
+    def get_ppkw_from_ppid(extr_ppid):
+        code, pp_extr_kwargs = extr_ppid.split(":")
+        if code != QueueEventExtractor.get_ppid_code():
+            raise ValueError(
+                f"Could not find extraction method '{code}'!")
+        kwargs = ppid_to_kwargs(cls=QueueEventExtractor,
+                                method="get_events_from_masks",
+                                ppid=pp_extr_kwargs)
+        return kwargs
 
     def process_label(self, label, index):
         """Process one label image, extracting masks and features"""
@@ -245,11 +281,7 @@ class QueueEventExtractor:
             # TODO: Do this before segmentation already?
             # skip events that have been analyzed already
             return None
-        if self.preselect:
-            # TODO: Do this before segmentation already?
-            ptp = np.ptp(self.data.image_corr[index])
-            if ptp < 0.1 * self.ptp_median:
-                return None
+
         masks = self.get_masks_from_label(label)
         if masks.size:
             events = self.get_events_from_masks(
@@ -300,14 +332,22 @@ class QueueEventExtractor:
                     self.event_queue.put((index, events))
 
         self.logger.debug(f"Finalizing `run` for PID {os.getpid()}, {self}")
-        # Explicitly close the event queue and join it
-        self.event_queue.close()
-        self.event_queue.join_thread()
-        self.logger.debug(f"End of `run` for PID {os.getpid()}, {self}")
-        # Also close the logging queue. Not that not all messages might
-        # arrive in the logging queue, since we called `cancel_join_thread`
-        # earlier.
-        self.log_queue.close()
+        if self.close_queues:
+            # Explicitly close the event queue and join it
+            self.event_queue.close()
+            self.event_queue.join_thread()
+            self.logger.debug(f"End of `run` for PID {os.getpid()}, {self}")
+            # Also close the logging queue. Note that not all messages might
+            # arrive in the logging queue, since we called `cancel_join_thread`
+            # earlier.
+            self.log_queue.close()
+
+    @classmethod
+    def get_ppid_from_kwargs(cls, kwargs):
+        warnings.warn(
+            "Please use get_ppid_from_ppkw instead of get_ppid_from_kwargs.",
+            DeprecationWarning)
+        return cls.get_ppid_from_ppkw(kwargs)
 
 
 class EventExtractorProcess(QueueEventExtractor, mp_spawn.Process):

dcnum/logic/__init__.py

@@ -0,0 +1,4 @@
+# flake8: noqa: F401
+"""Logic for running the dcnum pipeline"""
+from .job import DCNumPipelineJob
+from .ctrl import DCNumJobRunner