dcnum 0.13.2__py3-none-any.whl → 0.23.1__py3-none-any.whl

dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.13.2'
-__version_tuple__ = version_tuple = (0, 13, 2)
+__version__ = version = '0.23.1'
+__version_tuple__ = version_tuple = (0, 23, 1)

dcnum/feat/__init__.py

@@ -1,5 +1,6 @@
 # flake8: noqa: F401
-from . import feat_background, feat_brightness, feat_moments, feat_texture
+"""Feature computation"""
+from . import feat_background, feat_brightness, feat_contour, feat_texture
 from .event_extractor_manager_thread import EventExtractorManagerThread
 from .queue_event_extractor import (
     QueueEventExtractor, EventExtractorThread, EventExtractorProcess

dcnum/feat/event_extractor_manager_thread.py

@@ -1,3 +1,5 @@
+"""Feature computation: managing event extraction threads"""
+import collections
 import logging
 import multiprocessing as mp
 import threading
@@ -16,6 +18,7 @@ class EventExtractorManagerThread(threading.Thread):
                  labels_list: List,
                  fe_kwargs: Dict,
                  num_workers: int,
+                 writer_dq: collections.deque,
                  debug: bool = False,
                  *args, **kwargs):
         """Manage event extraction threads or precesses
@@ -39,9 +42,12 @@ class EventExtractorManagerThread(threading.Thread):
             :func:`.EventExtractor.get_init_kwargs` for more information.
         num_workers:
             Number of child threads or worker processes to use.
+        writer_dq:
+            The queue the writer uses. We monitor this queue. If it
+            fills up, we take a break.
         debug:
-            Whether to run in debugging mode which means more log
-            messages and only one thread (`num_workers` has no effect).
+            Whether to run in debugging mode which means only one
+            event extraction thread (`num_workers` has no effect).
         """
         super(EventExtractorManagerThread, self).__init__(
               name="EventExtractorManager", *args, **kwargs)
@@ -65,6 +71,8 @@ class EventExtractorManagerThread(threading.Thread):
         self.label_array = np.ctypeslib.as_array(
             self.fe_kwargs["label_array"]).reshape(
             self.data.image.chunk_shape)
+        #: Writer deque to monitor
+        self.writer_dq = writer_dq
         #: Time counter for feature extraction
         self.t_count = 0
         #: Whether debugging is enabled
@@ -76,30 +84,52 @@ class EventExtractorManagerThread(threading.Thread):
             worker_cls = EventExtractorThread
         else:
             worker_cls = EventExtractorProcess
-        workers = [worker_cls(*list(self.fe_kwargs.values()))
-                   for _ in range(self.num_workers)]
+        workers = [worker_cls(*list(self.fe_kwargs.values()), worker_index=ii)
+                   for ii in range(self.num_workers)]
         [w.start() for w in workers]
+        worker_monitor = self.fe_kwargs["worker_monitor"]
 
+        num_slots = len(self.slot_states)
         chunks_processed = 0
+        frames_processed = 0
         while True:
-            num_slots = len(self.slot_states)
-            cur_slot = 0
+            # If the writer_dq starts filling up, then this could lead to
+            # an oom-kill signal. Stall for the writer to prevent this.
+            if (ldq := len(self.writer_dq)) > 1000:
+                time.sleep(1)
+                ldq2 = len(self.writer_dq)
+                stall_time = max(0., (ldq2 - 200) / ((ldq - ldq2) or 1))
+                time.sleep(stall_time)
+                self.logger.warning(
+                    f"Stalled {stall_time + 1:.1f}s for slow writer "
+                    f"({ldq} chunks queued)")
+
             unavailable_slots = 0
+            found_free_slot = False
             # Check all slots for segmented labels
-            while True:
-                # - "e" there is data from the segmenter (the extractor
-                #   can take it and process it)
-                # - "s" the extractor processed the data and is waiting
-                #   for the segmenter
-                if self.slot_states[cur_slot] == "e":
-                    break
-                else:
-                    unavailable_slots += 1
-                    cur_slot = (cur_slot + 1) % num_slots
-                if unavailable_slots >= num_slots:
-                    # There is nothing to do, try to avoid 100% CPU
-                    unavailable_slots = 0
-                    time.sleep(.1)
+            while not found_free_slot:
+                # We sort the slots according to the slot chunks so that we
+                # always process the slot with the smallest slot chunk number
+                # first. Initially, the slot_chunks array is filled with
+                # zeros, but the segmenter fills up the slots with the lowest
+                # number first.
+                for cur_slot in np.argsort(self.slot_chunks):
+                    # - "e" there is data from the segmenter (the extractor
+                    #   can take it and process it)
+                    # - "s" the extractor processed the data and is waiting
+                    #   for the segmenter
+                    if self.slot_states[cur_slot] == "e":
+                        # The segmenter has something for us in this slot.
+                        found_free_slot = True
+                        break
+                    else:
+                        # Try another slot.
+                        unavailable_slots += 1
+                        cur_slot = (cur_slot + 1) % num_slots
+                    if unavailable_slots >= num_slots:
+                        # There is nothing to do, try to avoid 100% CPU
+                        unavailable_slots = 0
+                        time.sleep(.1)
 
             t1 = time.monotonic()
 
@@ -114,34 +144,38 @@ class EventExtractorManagerThread(threading.Thread):
                 self.label_array[len(new_labels):] = 0
             else:
                 raise ValueError("labels_list contains bad size data!")
+
             # Let the workers know there is work
-            for ii in range(self.data.image.get_chunk_size(chunk)):
-                self.raw_queue.put((chunk, ii))
+            chunk_size = self.data.image.get_chunk_size(chunk)
+            [self.raw_queue.put((chunk, ii)) for ii in range(chunk_size)]
 
             # Make sure the entire chunk has been processed.
-            while self.raw_queue.qsize():
+            while np.sum(worker_monitor) != frames_processed + chunk_size:
                 time.sleep(.1)
 
             # We are done here. The segmenter may continue its deed.
             self.slot_states[cur_slot] = "w"
 
-            self.logger.debug(f"Extracted one chunk: {chunk}")
+            self.logger.debug(f"Extracted chunk {chunk} in slot {cur_slot}")
             self.t_count += time.monotonic() - t1
 
             chunks_processed += 1
+            frames_processed += chunk_size
 
             if chunks_processed == self.data.image.num_chunks:
                 break
 
-        self.logger.debug("Waiting for event_queue to empty.")
-        # Wait until the event queue is empty.
-        event_queue = self.fe_kwargs["event_queue"]
-        while not event_queue.empty():
-            # The collector thread is still sorting things out. Wait
-            # before joining the threads.
-            time.sleep(.1)
-        self.logger.debug("Requesting extraction workers to join.")
+        inv_masks = self.fe_kwargs["invalid_mask_counter"].value
+        if inv_masks:
+            self.logger.info(f"Encountered {inv_masks} invalid masks")
+            inv_frac = inv_masks / len(self.data)
+            if inv_frac > 0.005:  # warn above one half percent
+                self.logger.warning(f"Discarded {inv_frac:.1%} of the masks, "
+                                    f"please check segmenter applicability")
+
+        self.logger.debug("Requesting extraction workers to join")
         self.fe_kwargs["finalize_extraction"].value = True
         [w.join() for w in workers]
-        self.logger.debug("Finished extraction.")
+
+        self.logger.debug("Finished extraction")
         self.logger.info(f"Extraction time: {self.t_count:.1f}s")

dcnum/feat/feat_background/__init__.py

@@ -1,16 +1,7 @@
 # flake8: noqa: F401
-import functools
-
-from .base import Background
+"""Feature computation: background image data from image data"""
+from .base import Background, get_available_background_methods
 # Background methods are registered by importing them here.
+from .bg_copy import BackgroundCopy
 from .bg_roll_median import BackgroundRollMed
 from .bg_sparse_median import BackgroundSparseMed
-
-
-@functools.cache
-def get_available_background_methods():
-    """Return dictionary of background computation methods"""
-    methods = {}
-    for cls in Background.__subclasses__():
-        methods[cls.key()] = cls
-    return methods

dcnum/feat/feat_background/base.py

@@ -1,16 +1,19 @@
 import abc
+import functools
 import inspect
 import multiprocessing as mp
 import pathlib
-import uuid
 
 import h5py
-import hdf5plugin
-import numpy as np
 
 from ...meta import ppid
 from ...read import HDF5Data
-from ...write import create_with_basins
+from ...write import HDF5Writer, create_with_basins, set_default_filter_kwargs
+
+
+# All subprocesses should use 'spawn' to avoid issues with threads
+# and 'fork' on POSIX systems.
+mp_spawn = mp.get_context('spawn')
 
 
 class Background(abc.ABC):
@@ -53,12 +56,14 @@ class Background(abc.ABC):
         self.kwargs.update(kwargs)
 
         if num_cpus is None:
-            num_cpus = mp.cpu_count()
+            num_cpus = mp_spawn.cpu_count()
         #: number of CPUs used
         self.num_cpus = num_cpus
 
-        #: number of frames
-        self.event_count = None
+        #: number of images in the input data
+        self.image_count = None
+        #: fraction images that have been processed
+        self.image_proc = mp_spawn.Value("d", 0)
 
         #: HDF5Data instance for input data
         self.hdin = None
@@ -86,12 +91,10 @@ class Background(abc.ABC):
         else:
             self.input_data = input_data
 
-        #: unique identifier
-        self.name = str(uuid.uuid4())
         #: shape of event images
         self.image_shape = self.input_data[0].shape
         #: total number of events
-        self.event_count = len(self.input_data)
+        self.image_count = len(self.input_data)
 
         if self.h5out is None:
             if not output_path.exists():
@@ -102,66 +105,23 @@ class Background(abc.ABC):
             # "a", because output file already exists
             self.h5out = h5py.File(output_path, "a", libver="latest")
 
-        # Initialize background data
-        if compress:
-            compression_kwargs = hdf5plugin.Zstd(clevel=5)
-        else:
-            compression_kwargs = {}
-        h5bg = self.h5out.require_dataset(
-            "events/image_bg",
-            shape=self.input_data.shape,
-            dtype=np.uint8,
-            chunks=(min(100, self.event_count),
-                    self.image_shape[0],
-                    self.image_shape[1]),
-            fletcher32=True,
-            **compression_kwargs,
+        # Initialize writer
+        self.writer = HDF5Writer(
+            obj=self.h5out,
+            ds_kwds=set_default_filter_kwargs(compression=compress),
         )
-        h5bg.attrs.create('CLASS', np.string_('IMAGE'))
-        h5bg.attrs.create('IMAGE_VERSION', np.string_('1.2'))
-        h5bg.attrs.create('IMAGE_SUBCLASS', np.string_('IMAGE_GRAYSCALE'))
 
     def __enter__(self):
         return self
 
     def __exit__(self, type, value, tb):
+        self.writer.close()
         # Close h5in and h5out
         if self.hdin is not None:  # we have an input file
             self.hdin.close()  # this closes self.h5in
         if self.h5in is not self.h5out and self.h5out is not None:
             self.h5out.close()
 
-    @staticmethod
-    def get_kwargs_from_ppid(bg_ppid):
-        """Return keyword arguments for any subclass from a PPID string"""
-        name, pp_check_user_kwargs = bg_ppid.split(":")
-        for cls in Background.__subclasses__():
-            if cls.key() == name:
-                break
-        else:
-            raise ValueError(
-                f"Could not find background computation method '{name}'!")
-        kwargs = ppid.ppid_to_kwargs(cls=cls,
-                                     method="check_user_kwargs",
-                                     ppid=pp_check_user_kwargs)
-        return kwargs
-
-    @classmethod
-    def get_ppid_from_kwargs(cls, kwargs):
-        """Return the PPID based on given keyword arguments for a subclass"""
-        key = cls.key()
-        cback = ppid.kwargs_to_ppid(cls, "check_user_kwargs", kwargs)
-        return ":".join([key, cback])
-
-    @classmethod
-    def key(cls):
-        if cls is Background:
-            raise ValueError("Cannot get `key` for `Background` base class!")
-        key = cls.__name__.lower()
-        if key.startswith("background"):
-            key = key[10:]
-        return key
-
     @abc.abstractmethod
     def check_user_kwargs(self, **kwargs):
         """Implement this to check the kwargs during init"""
@@ -170,7 +130,7 @@ class Background(abc.ABC):
         """Return a unique background pipeline identifier
 
         The pipeline identifier is universally applicable and must
-        be backwards-compatible (future versions of dcevent will
+        be backwards-compatible (future versions of dcnum will
         correctly acknowledge the ID).
 
         The segmenter pipeline ID is defined as::
@@ -186,17 +146,72 @@ class Background(abc.ABC):
 
             k=100^b=10000
         """
-        return self.get_ppid_from_kwargs(self.kwargs)
+        return self.get_ppid_from_ppkw(self.kwargs)
+
+    @classmethod
+    def get_ppid_code(cls):
+        if cls is Background:
+            raise ValueError("Cannot get `key` for `Background` base class!")
+        key = cls.__name__.lower()
+        if key.startswith("background"):
+            key = key[10:]
+        return key
+
+    @classmethod
+    def get_ppid_from_ppkw(cls, kwargs):
+        """Return the PPID based on given keyword arguments for a subclass"""
+        code = cls.get_ppid_code()
+        cback = ppid.kwargs_to_ppid(cls, "check_user_kwargs", kwargs)
+        return ":".join([code, cback])
+
+    @staticmethod
+    def get_ppkw_from_ppid(bg_ppid):
+        """Return keyword arguments for any subclass from a PPID string"""
+        code, pp_check_user_kwargs = bg_ppid.split(":")
+        for bg_code in get_available_background_methods():
+            if bg_code == code:
+                cls = get_available_background_methods()[bg_code]
+                break
+        else:
+            raise ValueError(
+                f"Could not find background computation method '{code}'!")
+        kwargs = ppid.ppid_to_kwargs(cls=cls,
+                                     method="check_user_kwargs",
+                                     ppid=pp_check_user_kwargs)
+        return kwargs
+
+    def get_progress(self):
+        """Return progress of background computation, float in [0,1]"""
+        if self.image_count == 0:
+            return 0.
+        else:
+            return self.image_proc.value
 
     def process(self):
+        # Delete any old background data
+        for key in ["image_bg", "bg_off"]:
+            if key in self.h5out["events"]:
+                del self.h5out["events"][key]
+        # Perform the actual background computation
         self.process_approach()
-
         bg_ppid = self.get_ppid()
-        # Store pipeline information in the image_bg feature
-        self.h5out["events/image_bg"].attrs["dcnum ppid background"] = bg_ppid
-        self.h5out["events/image_bg"].attrs["dcnum ppid generation"] = \
-            ppid.DCNUM_PPID_GENERATION
+        # Store pipeline information in the image_bg/bg_off feature
+        for key in ["image_bg", "bg_off"]:
+            if key in self.h5out["events"]:
+                self.h5out[f"events/{key}"].attrs["dcnum ppid background"] = \
+                    bg_ppid
+                self.h5out[F"events/{key}"].attrs["dcnum ppid generation"] = \
+                    ppid.DCNUM_PPID_GENERATION
 
     @abc.abstractmethod
     def process_approach(self):
         """The actual background computation approach"""
+
+
+@functools.cache
+def get_available_background_methods():
+    """Return dictionary of background computation methods"""
+    methods = {}
+    for cls in Background.__subclasses__():
+        methods[cls.get_ppid_code()] = cls
+    return methods

dcnum/feat/feat_background/bg_copy.py

@@ -0,0 +1,31 @@
+import h5py
+
+from .base import Background
+
+
+class BackgroundCopy(Background):
+    @staticmethod
+    def check_user_kwargs():
+        pass
+
+    def process(self):
+        """Copy input data to output dataset"""
+        if self.h5in != self.h5out:
+            hin = self.hdin.h5
+            for feat in ["image_bg", "bg_off"]:
+                if feat in hin["events"]:
+                    h5py.h5o.copy(src_loc=hin["events"].id,
+                                  src_name=feat.encode("utf-8"),
+                                  dst_loc=self.h5out["events"].id,
+                                  dst_name=feat.encode("utf-8"),
+                                  )
+
+        # set progress to 100%
+        self.image_proc.value = 1
+
+    def process_approach(self):
+        # We do the copying in `process`, because we do not want to modify
+        # any metadata or delete datasets as is done in the base class.
+        # But we still have to implement this method, because it is an
+        # abstractmethod in the base class.
+        pass

dcnum/feat/feat_background/bg_roll_median.py

@@ -1,16 +1,10 @@
-import multiprocessing as mp
 import queue
 import time
 
 import numpy as np
 from scipy import ndimage
 
-from .base import Background
-
-
-# All subprocesses should use 'spawn' to avoid issues with threads
-# and 'fork' on POSIX systems.
-mp_spawn = mp.get_context('spawn')
+from .base import mp_spawn, Background
 
 
 class BackgroundRollMed(Background):
@@ -63,6 +57,11 @@ class BackgroundRollMed(Background):
             kernel_size=kernel_size,
             batch_size=batch_size)
 
+        if kernel_size > len(self.input_data):
+            raise ValueError(f"Cannot compute background when the input data "
+                             f"size {len(self.input_data)} is larger than the "
+                             f"kernel size {kernel_size}!")
+
         #: kernel size used for median filtering
         self.kernel_size = kernel_size
         #: number of events processed at once
@@ -96,12 +95,12 @@ class BackgroundRollMed(Background):
         #: queue for median computation jobs
         self.queue = mp_spawn.Queue()
         #: list of workers (processes)
-        self.workers = [MedianWorker(self.queue,
-                                     self.worker_counter,
-                                     self.shared_input_raw,
-                                     self.shared_output_raw,
-                                     self.batch_size,
-                                     self.kernel_size)
+        self.workers = [WorkerRollMed(self.queue,
+                                      self.worker_counter,
+                                      self.shared_input_raw,
+                                      self.shared_output_raw,
+                                      self.batch_size,
+                                      self.kernel_size)
                         for _ in range(self.num_cpus)]
         [w.start() for w in self.workers]
 
@@ -120,7 +119,7 @@ class BackgroundRollMed(Background):
         """Check user-defined properties of this class
 
         This method primarily exists so that the CLI knows which
-        keyword arguements can be passed to this class.
+        keyword arguments can be passed to this class.
 
         Parameters
         ----------
@@ -133,7 +132,8 @@ class BackgroundRollMed(Background):
             `kernel_size` will not increase computation speed. Larger
             values lead to a higher memory consumption.
         """
-        assert kernel_size > 0
+        assert kernel_size > 0, "kernel size must be positive number"
+        assert kernel_size % 2 == 0, "kernel size must be even number"
         assert batch_size > kernel_size
 
     def get_slices_for_batch(self, batch_index=0):
@@ -147,9 +147,9 @@ class BackgroundRollMed(Background):
         stop_in = (batch_index + 1) * self.batch_size + self.kernel_size
         stop_out = (batch_index + 1) * self.batch_size
 
-        if stop_in > self.event_count:
-            stop_in = self.event_count
-            stop_out = self.event_count - self.kernel_size
+        if stop_in > self.image_count:
+            stop_in = self.image_count
+            stop_out = self.image_count - self.kernel_size
 
         slice_in = slice(start, stop_in)
         slice_out = slice(start, stop_out)
@@ -170,16 +170,21 @@ class BackgroundRollMed(Background):
 
     def process_approach(self):
         """Perform median computation on entire input data"""
-        num_steps = int(np.ceil(self.event_count / self.batch_size))
+        num_steps = int(np.ceil(self.image_count / self.batch_size))
         for ii in range(num_steps):
-            print(f"Computing background {ii/num_steps*100:.0f}%",
-                  end="\r", flush=True)
             self.process_next_batch()
-        # Set the remaining kernel_size median values to the last one
-        last_image = self.h5out["events/image_bg"][-self.kernel_size-1]
-        for ii in range(self.kernel_size):
-            self.h5out["events/image_bg"][self.event_count-ii-1] = last_image
-        print("Computing background 100%    ", flush=True)
+
+        # Set the remaining median bg images to the last one.
+        num_remaining = (self.input_data.shape[0]
+                         - self.h5out["events/image_bg"].shape[0])
+        if num_remaining:
+            last_image = self.h5out["events/image_bg"][-1]
+            last_chunk = np.repeat(
+                last_image[np.newaxis],
+                num_remaining,
+                axis=0)
+            self.writer.store_feature_chunk("image_bg", last_chunk)
+        self.image_proc.value = 1
 
     def process_next_batch(self):
         """Process one batch of input data"""
@@ -211,18 +216,21 @@ class BackgroundRollMed(Background):
             # TODO:
             #  Do this in a different thread so workers can keep going
             #  and use a lock somewhere in case the disk is too slow.
-            self.h5out["events/image_bg"][cur_slice_out] = \
+            self.writer.store_feature_chunk(
+                "image_bg",
                 self.shared_output[:output_size].reshape(output_size,
-                                                         *self.image_shape)
+                                                         *self.image_shape),
+            )
 
         self.current_batch += 1
+        self.image_proc.value += self.batch_size / self.image_count
 
 
-class MedianWorker(mp_spawn.Process):
+class WorkerRollMed(mp_spawn.Process):
     def __init__(self, job_queue, counter, shared_input, shared_output,
                  batch_size, kernel_size, *args, **kwargs):
         """Worker process for median computation"""
-        super(MedianWorker, self).__init__(*args, **kwargs)
+        super(WorkerRollMed, self).__init__(*args, **kwargs)
         self.queue = job_queue
         self.queue.cancel_join_thread()
         self.counter = counter