dcnum 0.20.3__py3-none-any.whl → 0.21.0__py3-none-any.whl

dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.20.3'
-__version_tuple__ = version_tuple = (0, 20, 3)
+__version__ = version = '0.21.0'
+__version_tuple__ = version_tuple = (0, 21, 0)

dcnum/feat/event_extractor_manager_thread.py

@@ -46,8 +46,8 @@ class EventExtractorManagerThread(threading.Thread):
             The queue the writer uses. We monitor this queue. If it
             fills up, we take a break.
         debug:
-            Whether to run in debugging mode which means more log
-            messages and only one thread (`num_workers` has no effect).
+            Whether to run in debugging mode which means only one
+            event extraction thread (`num_workers` has no effect).
         """
         super(EventExtractorManagerThread, self).__init__(
               name="EventExtractorManager", *args, **kwargs)
@@ -104,25 +104,32 @@ class EventExtractorManagerThread(threading.Thread):
                     f"Stalled {stall_time + 1:.1f}s for slow writer "
                     f"({ldq} chunks queued)")
 
-            cur_slot = 0
             unavailable_slots = 0
+            found_free_slot = False
             # Check all slots for segmented labels
-            while True:
-                # - "e" there is data from the segmenter (the extractor
-                #   can take it and process it)
-                # - "s" the extractor processed the data and is waiting
-                #   for the segmenter
-                if self.slot_states[cur_slot] == "e":
-                    # The segmenter has something for us in this slot.
-                    break
-                else:
-                    # Try another slot.
-                    unavailable_slots += 1
-                    cur_slot = (cur_slot + 1) % num_slots
-                if unavailable_slots >= num_slots:
-                    # There is nothing to do, try to avoid 100% CPU
-                    unavailable_slots = 0
-                    time.sleep(.1)
+            while not found_free_slot:
+                # We sort the slots according to the slot chunks so that we
+                # always process the slot with the smallest slot chunk number
+                # first. Initially, the slot_chunks array is filled with
+                # zeros, but the segmenter fills up the slots with the lowest
+                # number first.
+                for cur_slot in np.argsort(self.slot_chunks):
+                    # - "e" there is data from the segmenter (the extractor
+                    #   can take it and process it)
+                    # - "s" the extractor processed the data and is waiting
+                    #   for the segmenter
+                    if self.slot_states[cur_slot] == "e":
+                        # The segmenter has something for us in this slot.
+                        found_free_slot = True
+                        break
+                    else:
+                        # Try another slot.
+                        unavailable_slots += 1
+                        cur_slot = (cur_slot + 1) % num_slots
+                    if unavailable_slots >= num_slots:
+                        # There is nothing to do, try to avoid 100% CPU
+                        unavailable_slots = 0
+                        time.sleep(.1)
 
             t1 = time.monotonic()
 
@@ -149,7 +156,7 @@ class EventExtractorManagerThread(threading.Thread):
             # We are done here. The segmenter may continue its deed.
             self.slot_states[cur_slot] = "w"
 
-            self.logger.debug(f"Extracted one chunk: {chunk}")
+            self.logger.debug(f"Extracted chunk {chunk} in slot {cur_slot}")
             self.t_count += time.monotonic() - t1
 
             chunks_processed += 1

dcnum/feat/queue_event_extractor.py

@@ -36,7 +36,7 @@ class QueueEventExtractor:
                  finalize_extraction: mp.Value,
                  invalid_mask_counter: mp.Value,
                  worker_monitor: mp.RawArray,
-                 log_level: int = logging.INFO,
+                 log_level: int = None,
                  extract_kwargs: dict = None,
                  worker_index: int = None,
                  *args, **kwargs):
@@ -103,7 +103,7 @@ class QueueEventExtractor:
         # it looks like we have the same PID as the parent process. We
         # are setting up logging in `run`.
         self.logger = None
-        self.log_level = log_level
+        self.log_level = log_level or logging.getLogger("dcnum").level
         #: Shared array of length `len(data)` into which the number of
         #: events per frame is written.
         self.feat_nevents = feat_nevents
@@ -124,7 +124,7 @@ class QueueEventExtractor:
                         gate: Gate,
                         num_extractors: int,
                         log_queue: mp.Queue,
-                        log_level: int = logging.INFO,
+                        log_level: int = None,
                         ):
         """Get initialization arguments for :cass:`.QueueEventExtractor`
 
@@ -172,7 +172,7 @@ class QueueEventExtractor:
         args["finalize_extraction"] = mp_spawn.Value("b", False)
         args["invalid_mask_counter"] = mp_spawn.Value("L", 0)
         args["worker_monitor"] = mp_spawn.RawArray("L", num_extractors)
-        args["log_level"] = log_level
+        args["log_level"] = log_level or logging.getLogger("dcnum").level
         return args
 
     def get_events_from_masks(self, masks, data_index, *,

dcnum/logic/ctrl.py

@@ -90,8 +90,7 @@ class DCNumJobRunner(threading.Thread):
         # Set up logging
         # General logger for this job
         self.main_logger = logging.getLogger("dcnum")
-        self.main_logger.setLevel(
-            logging.DEBUG if job["debug"] else logging.INFO)
+        self.main_logger.setLevel(job["log_level"])
         # Log file output in target directory
         self.path_log = job["path_out"].with_suffix(".log")
         self.path_log.parent.mkdir(exist_ok=True, parents=True)
@@ -659,15 +658,24 @@ class DCNumJobRunner(threading.Thread):
             # Split segmentation and feature extraction workers evenly.
             num_extractors = self.job["num_procs"] // 2
             num_segmenters = self.job["num_procs"] - num_extractors
+            # leave one CPU for the writer and the remaining Threads
+            num_segmenters -= 1
         else:  # GPU segmenter
             num_slots = 3
             num_extractors = self.job["num_procs"]
+            # leave one CPU for the writer and the remaining Threads
+            num_extractors -= 1
             num_segmenters = 1
         num_extractors = max(1, num_extractors)
         num_segmenters = max(1, num_segmenters)
         self.job.kwargs["segmenter_kwargs"]["num_workers"] = num_segmenters
-        slot_chunks = mp_spawn.Array("i", num_slots)
-        slot_states = mp_spawn.Array("u", num_slots)
+        self.job.kwargs["segmenter_kwargs"]["debug"] = self.job["debug"]
+        slot_chunks = mp_spawn.Array("i", num_slots, lock=False)
+        slot_states = mp_spawn.Array("u", num_slots, lock=False)
+
+        self.logger.debug(f"Number of slots: {num_slots}")
+        self.logger.debug(f"Number of segmenters: {num_segmenters}")
+        self.logger.debug(f"Number of extractors: {num_extractors}")
 
         # Initialize segmenter manager thread
         thr_segm = SegmenterManagerThread(
@@ -676,7 +684,6 @@ class DCNumJobRunner(threading.Thread):
             bg_off=self.dtin["bg_off"] if "bg_off" in self.dtin else None,
             slot_states=slot_states,
             slot_chunks=slot_chunks,
-            debug=self.job["debug"],
         )
         thr_segm.start()
 
@@ -686,7 +693,7 @@ class DCNumJobRunner(threading.Thread):
             gate=gate.Gate(self.dtin, **self.job["gate_kwargs"]),
             num_extractors=num_extractors,
             log_queue=self.log_queue,
-            log_level=logging.DEBUG if self.job["debug"] else logging.INFO,
+            log_level=self.logger.level,
             )
         fe_kwargs["extract_kwargs"] = self.job["feature_kwargs"]
 

dcnum/logic/job.py

@@ -1,6 +1,7 @@
 import collections
 import copy
 import inspect
+import logging
 import multiprocessing as mp
 import pathlib
 from typing import Dict, Literal
@@ -31,6 +32,7 @@ class DCNumPipelineJob:
                  basin_strategy: Literal["drain", "tap"] = "drain",
                  no_basins_in_output: bool = None,
                  num_procs: int = None,
+                 log_level: int = logging.INFO,
                  debug: bool = False,
                  ):
         """Pipeline job recipe
@@ -73,8 +75,11 @@ class DCNumPipelineJob:
             Deprecated
         num_procs: int
             Number of processes to use
+        log_level: int
+            Logging level to use.
         debug: bool
-            Whether to be verbose and use threads instead of processes
+            Whether to set logging level to "DEBUG" and
+            use threads instead of processes
         """
         if no_basins_in_output is not None:
             warnings.warn("The `no_basins_in_output` keyword argument is "
@@ -104,6 +109,9 @@ class DCNumPipelineJob:
         if path_out is None:
             pin = pathlib.Path(path_in)
             path_out = pin.with_name(pin.stem + "_dcn.rtdc")
+        # Set logging level to DEBUG in debugging mode
+        if self.kwargs["debug"]:
+            self.kwargs["log_level"] = logging.DEBUG
         self.kwargs["path_out"] = pathlib.Path(path_out)
         # Set default mask kwargs for segmenter
         self.kwargs["segmenter_kwargs"].setdefault("kwargs_mask", {})

dcnum/meta/ppid.py

@@ -10,7 +10,7 @@ import warnings
 
 #: Increment this string if there are breaking changes that make
 #: previous pipelines unreproducible.
-DCNUM_PPID_GENERATION = "8"
+DCNUM_PPID_GENERATION = "9"
 
 
 class ClassWithPPIDCapabilities(Protocol):

dcnum/segm/segmenter.py

@@ -19,7 +19,7 @@ class Segmenter(abc.ABC):
     #: Whether to enable mask post-processing. If disabled, you should
     #: make sure that your mask is properly defined and cleaned or you
     #: have to call `process_mask` in your `segment_approach` implementation.
-    mask_postprocessing = False
+    mask_postprocessing = True
     #: Default keyword arguments for mask post-processing. See `process_mask`
     #: for available options.
     mask_default_kwargs = {}
@@ -38,7 +38,7 @@ class Segmenter(abc.ABC):
         kwargs_mask: dict
             Keyword arguments for mask post-processing (see `process_mask`)
         debug: bool
-            Debugging parameters
+            Enable debugging mode (e.g. CPU segmenter runs in one thread)
         kwargs:
             Additional, optional keyword arguments for `segment_batch`.
         """

dcnum/segm/segmenter_cpu.py

@@ -189,9 +189,11 @@ class CPUSegmenter(Segmenter, abc.ABC):
         if self.debug:
             worker_cls = CPUSegmenterWorkerThread
             num_workers = 1
+            self.logger.debug("Running with one worker in main thread")
         else:
             worker_cls = CPUSegmenterWorkerProcess
             num_workers = min(self.num_workers, image_data.shape[0])
+            self.logger.debug(f"Running with {num_workers} workers")
 
         if not self._mp_workers:
             step_size = batch_size // num_workers

dcnum/segm/segmenter_gpu.py

@@ -10,7 +10,6 @@ from .segmenter import Segmenter
 
 class GPUSegmenter(Segmenter, abc.ABC):
     hardware_processor = "gpu"
-    mask_postprocessing = False
 
     def __init__(self,
                  *,

dcnum/segm/segmenter_manager_thread.py

@@ -18,7 +18,6 @@ class SegmenterManagerThread(threading.Thread):
                  slot_states: mp.Array,
                  slot_chunks: mp.Array,
                  bg_off: np.ndarray = None,
-                 debug: bool = False,
                  *args, **kwargs):
         """Manage the segmentation of image data
 
@@ -43,10 +42,6 @@ class SegmenterManagerThread(threading.Thread):
             1d array containing additional background image offset values
             that are added to each background image before subtraction
             from the input image
-        debug:
-            Whether to run in debugging mode (more verbose messages and
-            CPU-based segmentation is done in one single thread instead
-            of in multiple subprocesses).
 
         Notes
         -----
@@ -80,33 +75,36 @@ class SegmenterManagerThread(threading.Thread):
         self.labels_list = [None] * len(self.slot_states)
         #: Time counter for segmentation
         self.t_count = 0
-        #: Whether running in debugging mode
-        self.debug = debug
 
     def run(self):
         num_slots = len(self.slot_states)
         # We iterate over all the chunks of the image data.
         for chunk in self.image_data.iter_chunks():
-            cur_slot = 0
-            empty_slots = 0
+            unavailable_slots = 0
+            found_free_slot = False
             # Wait for a free slot to perform segmentation (compute labels)
-            while True:
-                # - "e" there is data from the segmenter (the extractor
-                #   can take it and process it)
-                # - "s" the extractor processed the data and is waiting
-                #   for the segmenter
-                if self.slot_states[cur_slot] != "e":
-                    # It's the segmenter's turn. Note that we use '!= "e"',
-                    # because the initial value is "\x00".
-                    break
-                else:
-                    # Try another slot.
-                    empty_slots += 1
-                    cur_slot = (cur_slot + 1) % num_slots
-                if empty_slots >= num_slots:
-                    # There is nothing to do, try to avoid 100% CPU
-                    empty_slots = 0
-                    time.sleep(.01)
+            while not found_free_slot:
+                # We sort the slots according to the slot chunks so that we
+                # always process the slot with the smallest slot chunk number
+                # first. Initially, the slot_chunks array is filled with
+                # zeros, but we populate it here.
+                for cur_slot in np.argsort(self.slot_chunks):
+                    # - "e" there is data from the segmenter (the extractor
+                    #   can take it and process it)
+                    # - "s" the extractor processed the data and is waiting
+                    #   for the segmenter
+                    if self.slot_states[cur_slot] != "e":
+                        # It's the segmenter's turn. Note that we use '!= "e"',
+                        # because the initial value is "\x00".
+                        found_free_slot = True
+                        break
+                    else:
+                        # Try another slot.
+                        unavailable_slots += 1
+                    if unavailable_slots >= num_slots:
+                        # There is nothing to do, try to avoid 100% CPU
+                        unavailable_slots = 0
+                        time.sleep(.1)
 
             t1 = time.monotonic()
 
@@ -125,7 +123,7 @@ class SegmenterManagerThread(threading.Thread):
             # This must be done last: Let the extractor know that this
             # slot is ready for processing.
             self.slot_states[cur_slot] = "e"
-            self.logger.debug(f"Segmented one chunk: {chunk}")
+            self.logger.debug(f"Segmented chunk {chunk} in slot {cur_slot}")
 
             self.t_count += time.monotonic() - t1
 

{dcnum-0.20.3.dist-info → dcnum-0.21.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.20.3
+Version: 0.21.0
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.20.3.dist-info → dcnum-0.21.0.dist-info}/RECORD

@@ -1,9 +1,9 @@
 dcnum/__init__.py,sha256=hcawIKS7utYiOyVhOAX9t7K3xYzP1b9862VV0b6qSrQ,74
-dcnum/_version.py,sha256=7ZBpIaoHyK5xe5B8niAFr7a4ME2vlFXrNQqhWCHU61Y,413
+dcnum/_version.py,sha256=ey0T2QT7DroTwq1OFrMquZ-sNrLzojab2ikTifByhUQ,413
 dcnum/feat/__init__.py,sha256=jUJYWTD3VIoDNKrmryXbjHb1rGwYtK4b7VPWihYgUoo,325
-dcnum/feat/event_extractor_manager_thread.py,sha256=fCU28Pavf71zMRV5-N_9Kn1q213RslnMS1UZRmKw9Uo,7284
+dcnum/feat/event_extractor_manager_thread.py,sha256=gmvWNMmMTuUENVbDAn59PEp4LiRDE0PQA4VYhfWQ8EY,7825
 dcnum/feat/gate.py,sha256=svbObmqpYdqPawpfrsEjTiUPJXf24GrNi8PXTKT-z44,7225
-dcnum/feat/queue_event_extractor.py,sha256=XhA930QVQ1Z4saisbcGTrEut-fSgwTyfDn6b9GzD4iY,15644
+dcnum/feat/queue_event_extractor.py,sha256=bNdYzMPto37FCIgBbBw-YRQ2TlTpJKCWj9r_Y4sak3E,15700
 dcnum/feat/feat_background/__init__.py,sha256=OTmMuazHNaSrZb2XW4cnJ6PlgJLbKrPbaidpEixYa0A,341
 dcnum/feat/feat_background/base.py,sha256=phZdyOrHQPjvYlw1JQ8DkdXw5H2-eE1LfLGqCAo1rlo,7965
 dcnum/feat/feat_background/bg_copy.py,sha256=PK8x4_Uph-_A6uszZC5uhe1gD1dSRdHnDMEsN0HSGHA,1034
@@ -20,12 +20,12 @@ dcnum/feat/feat_texture/__init__.py,sha256=6StM9S540UVtdFFR3bHa7nfCTomeVdoo7Uy9C
 dcnum/feat/feat_texture/common.py,sha256=COXHpXS-7DMouGu3WF83I76L02Sr7P9re4lxajh6g0E,439
 dcnum/feat/feat_texture/tex_all.py,sha256=eGjjNfPpfZw7FA_VNFCIMiU38KD0qcGbxLciYy-tCiA,4097
 dcnum/logic/__init__.py,sha256=7J3GrwJInNQbrLk61HRIV7X7p69TAIbMYpR34hh6u14,177
-dcnum/logic/ctrl.py,sha256=QU-Mwb_YKu-4oIk6XZ4oyXvzIeSc4CZodFVyIGqJnWs,34411
-dcnum/logic/job.py,sha256=cF4bPiEy5UkDmQN91Ku2yxBW0nXBEmKTNkIHlL2LT-U,6724
+dcnum/logic/ctrl.py,sha256=aqbYi3OsqcjATCXAhmTSrOUwyJRS9evOJuHZcdS0sDg,34792
+dcnum/logic/job.py,sha256=H1uDZ1nnNHNWWCe6mS8OWB0Uxc6XUKLISx5xExeplZY,7015
 dcnum/logic/json_encoder.py,sha256=cxMnqisbKEVf-rVcw6rK2BBAb6iz_hKFaGl81kK36lQ,571
 dcnum/meta/__init__.py,sha256=AVqRgyKXO1orKnE305h88IBvoZ1oz6X11HN1WP5nGvg,60
 dcnum/meta/paths.py,sha256=J_ikeHzd7gEeRgAKjuayz3x6q4h1fOiDadM-ZxhAGm4,1053
-dcnum/meta/ppid.py,sha256=Q3jg8lZt5tlGIby_-7rBqTANesMjJrmxASXZhsvBD_Y,7706
+dcnum/meta/ppid.py,sha256=2Lw7sSDFMZFaAiKM7kLB7ozBQJru8rV1DbUKdU5g5w8,7706
 dcnum/read/__init__.py,sha256=ksLdV8EkOU3EPje8teCOSehcUeGAZfg9TQ5ltuEUgls,216
 dcnum/read/cache.py,sha256=lisrGG7AyvVitf0h92wh5FvYCsxa0pWyGcAyYwGP-LQ,6471
 dcnum/read/const.py,sha256=GG9iyXDtEldvJYOBnhZjlimzIeBMAt4bSr2-xn2gzzc,464
@@ -33,16 +33,16 @@ dcnum/read/hdf5_data.py,sha256=Yyq02UTILc5ZgIQXpR9Y0wuX2WT8s0g23PraI7KxmJY,23489
 dcnum/read/mapped.py,sha256=UryArlrIsHxjOyimBL2Nooi3r73zuGtnGdqdxa6PK_g,3076
 dcnum/segm/__init__.py,sha256=iiq_1A9DU5wMUcKnsZ53E7NyzCkbZCJeUDimzunE-OM,247
 dcnum/segm/segm_thresh.py,sha256=W9f9--aYmXpFjv3IASB6I90V40PJcIe_Jt_1iK-eHhw,946
-dcnum/segm/segmenter.py,sha256=gVzmP6CuwI9Qfk8GN_xWGu_xbtVTOhxIOWn-2yr_H1Y,12220
-dcnum/segm/segmenter_cpu.py,sha256=IzhPNQaO4TBh3EzZqLGaBAeRryfBKnld7Joe8qY4AB4,10690
-dcnum/segm/segmenter_gpu.py,sha256=Au1MQdAalVsmJ-cmb3OcCmEMBfXSDuJjdXJTGqEIcG8,1962
-dcnum/segm/segmenter_manager_thread.py,sha256=xQEioOkASlm8DTdG0RBtjCJP1cOuiyJAm4q2n1l_tfM,5710
+dcnum/segm/segmenter.py,sha256=aCu7WfwOXU6nza1m3sAW7WhywR_jz3kmyfF3NCQo24A,12260
+dcnum/segm/segmenter_cpu.py,sha256=e6XTlEUqXR2HBKZJxumpCCHprdX8Ql7dsbIcfdf4dH4,10831
+dcnum/segm/segmenter_gpu.py,sha256=opgIBSL9lYFOo405xGoLUt2GuxpVQPMlPxcNHpQVt3s,1930
+dcnum/segm/segmenter_manager_thread.py,sha256=LbNBg1R19GaDKow8pgfOFx1sE6ulS27Ky47_pJBvnK4,5850
 dcnum/write/__init__.py,sha256=QvWHeZmjHI18i-YlGYuzN3i7dVWY9UCReKchrJ-gif0,260
 dcnum/write/deque_writer_thread.py,sha256=ao7F1yrVKyufgC4rC0Y2_Vt7snuT6KpI7W2qVxcjdhk,1994
 dcnum/write/queue_collector_thread.py,sha256=d_WfdsZdFnFsiAY0zVMwUlA4juIMeiWYmE_-rezBQCE,11734
 dcnum/write/writer.py,sha256=e6J8YVqhS7kzkpPIMoDMokJpqSy1WWNdOrwaJof1oVc,15601
-dcnum-0.20.3.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
-dcnum-0.20.3.dist-info/METADATA,sha256=vSymdBOf42F_8WQKPFJ1Bbc6EUeQkWqlCg-g3ikiXOY,2194
-dcnum-0.20.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-dcnum-0.20.3.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
-dcnum-0.20.3.dist-info/RECORD,,
+dcnum-0.21.0.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
+dcnum-0.21.0.dist-info/METADATA,sha256=CUlOprvoa-ThjgNoDt_liTNSR1Ao3hBdjb5J4FDn_Fo,2194
+dcnum-0.21.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+dcnum-0.21.0.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
+dcnum-0.21.0.dist-info/RECORD,,