dcnum 0.20.2__tar.gz → 0.20.4__tar.gz

{dcnum-0.20.2 → dcnum-0.20.4}/CHANGELOG

@@ -1,3 +1,11 @@
+0.20.4
+ - fix: slot_chunks were not performed in increasing order, making it
+   impossible for writer to continue writing and consequent OOM
+ - enh: reserve one CPU for writer thread and control logic
+ - enh: disentangle logging and debugging keyword arguments
+ - enh: do not initialize slot_chunks and slot_states with lock
+0.20.3
+ - enh: improve strategy for stalling for slow writer
 0.20.2
  - ref: remove __init__ from SegmentThresh
 0.20.1
@@ -24,7 +32,7 @@
    postprocessing
  - feat: allow to specify ranges when creating an HDF5Data instance to
    enable e.g. processing only a portion of an input file
- - feat: volume computation via contour revolve algorithm
+ - feat: volume computation via contour revolve algorithm (#23)
  - feat: background offset (flickering) correction via the
    "offset_correction" keyword for the "sparsemed" background computer
    and "bg_off" everywhere else

{dcnum-0.20.2/src/dcnum.egg-info → dcnum-0.20.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.20.2
+Version: 0.20.4
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.20.2 → dcnum-0.20.4}/src/dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.20.2'
-__version_tuple__ = version_tuple = (0, 20, 2)
+__version__ = version = '0.20.4'
+__version_tuple__ = version_tuple = (0, 20, 4)

{dcnum-0.20.2 → dcnum-0.20.4}/src/dcnum/feat/event_extractor_manager_thread.py

@@ -46,8 +46,8 @@ class EventExtractorManagerThread(threading.Thread):
             The queue the writer uses. We monitor this queue. If it
             fills up, we take a break.
         debug:
-            Whether to run in debugging mode which means more log
-            messages and only one thread (`num_workers` has no effect).
+            Whether to run in debugging mode which means only one
+            event extraction thread (`num_workers` has no effect).
         """
         super(EventExtractorManagerThread, self).__init__(
               name="EventExtractorManager", *args, **kwargs)
@@ -95,32 +95,41 @@ class EventExtractorManagerThread(threading.Thread):
         while True:
             # If the writer_dq starts filling up, then this could lead to
             # an oom-kill signal. Stall for the writer to prevent this.
-            ldq = len(self.writer_dq)
-            if ldq > 1000:
-                stallsec = ldq / 1000
+            if (ldq := len(self.writer_dq)) > 1000:
+                time.sleep(1)
+                ldq2 = len(self.writer_dq)
+                stall_time = (ldq2 - 200) / (ldq - ldq2) if ldq2 > 200 else 0
+                time.sleep(stall_time)
                 self.logger.warning(
-                    f"Stalling {stallsec:.1f}s for slow writer")
-                time.sleep(stallsec)
+                    f"Stalled {stall_time + 1:.1f}s for slow writer "
+                    f"({ldq} chunks queued)")
 
-            cur_slot = 0
             unavailable_slots = 0
+            found_free_slot = False
             # Check all slots for segmented labels
-            while True:
-                # - "e" there is data from the segmenter (the extractor
-                #   can take it and process it)
-                # - "s" the extractor processed the data and is waiting
-                #   for the segmenter
-                if self.slot_states[cur_slot] == "e":
-                    # The segmenter has something for us in this slot.
-                    break
-                else:
-                    # Try another slot.
-                    unavailable_slots += 1
-                    cur_slot = (cur_slot + 1) % num_slots
-                if unavailable_slots >= num_slots:
-                    # There is nothing to do, try to avoid 100% CPU
-                    unavailable_slots = 0
-                    time.sleep(.1)
+            while not found_free_slot:
+                # We sort the slots according to the slot chunks so that we
+                # always process the slot with the smallest slot chunk number
+                # first. Initially, the slot_chunks array is filled with
+                # zeros, but the segmenter fills up the slots with the lowest
+                # number first.
+                for cur_slot in np.argsort(self.slot_chunks):
+                    # - "e" there is data from the segmenter (the extractor
+                    #   can take it and process it)
+                    # - "s" the extractor processed the data and is waiting
+                    #   for the segmenter
+                    if self.slot_states[cur_slot] == "e":
+                        # The segmenter has something for us in this slot.
+                        found_free_slot = True
+                        break
+                    else:
+                        # Try another slot.
+                        unavailable_slots += 1
+                        cur_slot = (cur_slot + 1) % num_slots
+                    if unavailable_slots >= num_slots:
+                        # There is nothing to do, try to avoid 100% CPU
+                        unavailable_slots = 0
+                        time.sleep(.1)
 
             t1 = time.monotonic()
 
@@ -147,7 +156,7 @@ class EventExtractorManagerThread(threading.Thread):
             # We are done here. The segmenter may continue its deed.
             self.slot_states[cur_slot] = "w"
 
-            self.logger.debug(f"Extracted one chunk: {chunk}")
+            self.logger.debug(f"Extracted chunk {chunk} in slot {cur_slot}")
             self.t_count += time.monotonic() - t1
 
             chunks_processed += 1

{dcnum-0.20.2 → dcnum-0.20.4}/src/dcnum/feat/queue_event_extractor.py

@@ -36,7 +36,7 @@ class QueueEventExtractor:
                  finalize_extraction: mp.Value,
                  invalid_mask_counter: mp.Value,
                  worker_monitor: mp.RawArray,
-                 log_level: int = logging.INFO,
+                 log_level: int = None,
                  extract_kwargs: dict = None,
                  worker_index: int = None,
                  *args, **kwargs):
@@ -103,7 +103,7 @@ class QueueEventExtractor:
         # it looks like we have the same PID as the parent process. We
         # are setting up logging in `run`.
         self.logger = None
-        self.log_level = log_level
+        self.log_level = log_level or logging.getLogger("dcnum").level
         #: Shared array of length `len(data)` into which the number of
         #: events per frame is written.
         self.feat_nevents = feat_nevents
@@ -124,7 +124,7 @@ class QueueEventExtractor:
                         gate: Gate,
                         num_extractors: int,
                         log_queue: mp.Queue,
-                        log_level: int = logging.INFO,
+                        log_level: int = None,
                         ):
         """Get initialization arguments for :cass:`.QueueEventExtractor`
 
@@ -172,7 +172,7 @@ class QueueEventExtractor:
         args["finalize_extraction"] = mp_spawn.Value("b", False)
         args["invalid_mask_counter"] = mp_spawn.Value("L", 0)
         args["worker_monitor"] = mp_spawn.RawArray("L", num_extractors)
-        args["log_level"] = log_level
+        args["log_level"] = log_level or logging.getLogger("dcnum").level
         return args
 
     def get_events_from_masks(self, masks, data_index, *,

{dcnum-0.20.2 → dcnum-0.20.4}/src/dcnum/logic/ctrl.py

@@ -90,8 +90,7 @@ class DCNumJobRunner(threading.Thread):
         # Set up logging
         # General logger for this job
         self.main_logger = logging.getLogger("dcnum")
-        self.main_logger.setLevel(
-            logging.DEBUG if job["debug"] else logging.INFO)
+        self.main_logger.setLevel(job["log_level"])
         # Log file output in target directory
         self.path_log = job["path_out"].with_suffix(".log")
         self.path_log.parent.mkdir(exist_ok=True, parents=True)
@@ -659,15 +658,24 @@ class DCNumJobRunner(threading.Thread):
             # Split segmentation and feature extraction workers evenly.
             num_extractors = self.job["num_procs"] // 2
             num_segmenters = self.job["num_procs"] - num_extractors
+            # leave one CPU for the writer and the remaining Threads
+            num_segmenters -= 1
         else:  # GPU segmenter
             num_slots = 3
             num_extractors = self.job["num_procs"]
+            # leave one CPU for the writer and the remaining Threads
+            num_extractors -= 1
             num_segmenters = 1
         num_extractors = max(1, num_extractors)
         num_segmenters = max(1, num_segmenters)
         self.job.kwargs["segmenter_kwargs"]["num_workers"] = num_segmenters
-        slot_chunks = mp_spawn.Array("i", num_slots)
-        slot_states = mp_spawn.Array("u", num_slots)
+        self.job.kwargs["segmenter_kwargs"]["debug"] = self.job["debug"]
+        slot_chunks = mp_spawn.Array("i", num_slots, lock=False)
+        slot_states = mp_spawn.Array("u", num_slots, lock=False)
+
+        self.logger.debug(f"Number of slots: {num_slots}")
+        self.logger.debug(f"Number of segmenters: {num_segmenters}")
+        self.logger.debug(f"Number of extractors: {num_extractors}")
 
         # Initialize segmenter manager thread
         thr_segm = SegmenterManagerThread(
@@ -676,7 +684,6 @@ class DCNumJobRunner(threading.Thread):
             bg_off=self.dtin["bg_off"] if "bg_off" in self.dtin else None,
             slot_states=slot_states,
             slot_chunks=slot_chunks,
-            debug=self.job["debug"],
         )
         thr_segm.start()
 
@@ -686,7 +693,7 @@ class DCNumJobRunner(threading.Thread):
             gate=gate.Gate(self.dtin, **self.job["gate_kwargs"]),
             num_extractors=num_extractors,
             log_queue=self.log_queue,
-            log_level=logging.DEBUG if self.job["debug"] else logging.INFO,
+            log_level=self.logger.level,
             )
         fe_kwargs["extract_kwargs"] = self.job["feature_kwargs"]
 

{dcnum-0.20.2 → dcnum-0.20.4}/src/dcnum/logic/job.py

@@ -1,6 +1,7 @@
 import collections
 import copy
 import inspect
+import logging
 import multiprocessing as mp
 import pathlib
 from typing import Dict, Literal
@@ -31,6 +32,7 @@ class DCNumPipelineJob:
                  basin_strategy: Literal["drain", "tap"] = "drain",
                  no_basins_in_output: bool = None,
                  num_procs: int = None,
+                 log_level: int = logging.INFO,
                  debug: bool = False,
                  ):
         """Pipeline job recipe
@@ -73,8 +75,11 @@ class DCNumPipelineJob:
             Deprecated
         num_procs: int
             Number of processes to use
+        log_level: int
+            Logging level to use.
         debug: bool
-            Whether to be verbose and use threads instead of processes
+            Whether to set logging level to "DEBUG" and
+            use threads instead of processes
         """
         if no_basins_in_output is not None:
             warnings.warn("The `no_basins_in_output` keyword argument is "
@@ -104,6 +109,9 @@ class DCNumPipelineJob:
         if path_out is None:
             pin = pathlib.Path(path_in)
             path_out = pin.with_name(pin.stem + "_dcn.rtdc")
+        # Set logging level to DEBUG in debugging mode
+        if self.kwargs["debug"]:
+            self.kwargs["log_level"] = logging.DEBUG
         self.kwargs["path_out"] = pathlib.Path(path_out)
         # Set default mask kwargs for segmenter
         self.kwargs["segmenter_kwargs"].setdefault("kwargs_mask", {})

{dcnum-0.20.2 → dcnum-0.20.4}/src/dcnum/segm/segmenter.py

@@ -38,7 +38,7 @@ class Segmenter(abc.ABC):
         kwargs_mask: dict
             Keyword arguments for mask post-processing (see `process_mask`)
         debug: bool
-            Debugging parameters
+            Enable debugging mode (e.g. CPU segmenter runs in one thread)
         kwargs:
             Additional, optional keyword arguments for `segment_batch`.
         """

{dcnum-0.20.2 → dcnum-0.20.4}/src/dcnum/segm/segmenter_cpu.py

@@ -189,9 +189,11 @@ class CPUSegmenter(Segmenter, abc.ABC):
         if self.debug:
             worker_cls = CPUSegmenterWorkerThread
             num_workers = 1
+            self.logger.debug("Running with one worker in main thread")
         else:
             worker_cls = CPUSegmenterWorkerProcess
             num_workers = min(self.num_workers, image_data.shape[0])
+            self.logger.debug(f"Running with {num_workers} workers")
 
         if not self._mp_workers:
             step_size = batch_size // num_workers

{dcnum-0.20.2 → dcnum-0.20.4}/src/dcnum/segm/segmenter_manager_thread.py

@@ -18,7 +18,6 @@ class SegmenterManagerThread(threading.Thread):
                  slot_states: mp.Array,
                  slot_chunks: mp.Array,
                  bg_off: np.ndarray = None,
-                 debug: bool = False,
                  *args, **kwargs):
         """Manage the segmentation of image data
 
@@ -43,10 +42,6 @@ class SegmenterManagerThread(threading.Thread):
             1d array containing additional background image offset values
             that are added to each background image before subtraction
             from the input image
-        debug:
-            Whether to run in debugging mode (more verbose messages and
-            CPU-based segmentation is done in one single thread instead
-            of in multiple subprocesses).
 
         Notes
         -----
@@ -80,33 +75,36 @@ class SegmenterManagerThread(threading.Thread):
         self.labels_list = [None] * len(self.slot_states)
         #: Time counter for segmentation
         self.t_count = 0
-        #: Whether running in debugging mode
-        self.debug = debug
 
     def run(self):
         num_slots = len(self.slot_states)
         # We iterate over all the chunks of the image data.
         for chunk in self.image_data.iter_chunks():
-            cur_slot = 0
-            empty_slots = 0
+            unavailable_slots = 0
+            found_free_slot = False
             # Wait for a free slot to perform segmentation (compute labels)
-            while True:
-                # - "e" there is data from the segmenter (the extractor
-                #   can take it and process it)
-                # - "s" the extractor processed the data and is waiting
-                #   for the segmenter
-                if self.slot_states[cur_slot] != "e":
-                    # It's the segmenter's turn. Note that we use '!= "e"',
-                    # because the initial value is "\x00".
-                    break
-                else:
-                    # Try another slot.
-                    empty_slots += 1
-                    cur_slot = (cur_slot + 1) % num_slots
-                if empty_slots >= num_slots:
-                    # There is nothing to do, try to avoid 100% CPU
-                    empty_slots = 0
-                    time.sleep(.01)
+            while not found_free_slot:
+                # We sort the slots according to the slot chunks so that we
+                # always process the slot with the smallest slot chunk number
+                # first. Initially, the slot_chunks array is filled with
+                # zeros, but we populate it here.
+                for cur_slot in np.argsort(self.slot_chunks):
+                    # - "e" there is data from the segmenter (the extractor
+                    #   can take it and process it)
+                    # - "s" the extractor processed the data and is waiting
+                    #   for the segmenter
+                    if self.slot_states[cur_slot] != "e":
+                        # It's the segmenter's turn. Note that we use '!= "e"',
+                        # because the initial value is "\x00".
+                        found_free_slot = True
+                        break
+                    else:
+                        # Try another slot.
+                        unavailable_slots += 1
+                    if unavailable_slots >= num_slots:
+                        # There is nothing to do, try to avoid 100% CPU
+                        unavailable_slots = 0
+                        time.sleep(.1)
 
             t1 = time.monotonic()
 
@@ -125,7 +123,7 @@ class SegmenterManagerThread(threading.Thread):
             # This must be done last: Let the extractor know that this
             # slot is ready for processing.
             self.slot_states[cur_slot] = "e"
-            self.logger.debug(f"Segmented one chunk: {chunk}")
+            self.logger.debug(f"Segmented chunk {chunk} in slot {cur_slot}")
 
             self.t_count += time.monotonic() - t1
 

{dcnum-0.20.2 → dcnum-0.20.4/src/dcnum.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.20.2
+Version: 0.20.4
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.20.2 → dcnum-0.20.4}/tests/test_feat_event_extractor_manager.py

@@ -33,7 +33,6 @@ def test_event_extractor_manager_thread():
         image_data=hd.image_corr,
         slot_states=slot_states,
         slot_chunks=slot_chunks,
-        debug=True,
     )
     thr_segm.start()