dcnum 0.16.6__py3-none-any.whl → 0.16.8__py3-none-any.whl

dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.16.6'
-__version_tuple__ = version_tuple = (0, 16, 6)
+__version__ = version = '0.16.8'
+__version_tuple__ = version_tuple = (0, 16, 8)

dcnum/feat/event_extractor_manager_thread.py

@@ -1,4 +1,5 @@
 """Feature computation: managing event extraction threads"""
+import collections
 import logging
 import multiprocessing as mp
 import threading
@@ -17,6 +18,7 @@ class EventExtractorManagerThread(threading.Thread):
                  labels_list: List,
                  fe_kwargs: Dict,
                  num_workers: int,
+                 writer_dq: collections.deque,
                  debug: bool = False,
                  *args, **kwargs):
         """Manage event extraction threads or precesses
@@ -40,6 +42,9 @@ class EventExtractorManagerThread(threading.Thread):
             :func:`.EventExtractor.get_init_kwargs` for more information.
         num_workers:
             Number of child threads or worker processes to use.
+        writer_dq:
+            The queue the writer uses. We monitor this queue. If it
+            fills up, we take a break.
         debug:
             Whether to run in debugging mode which means more log
             messages and only one thread (`num_workers` has no effect).
@@ -66,6 +71,8 @@ class EventExtractorManagerThread(threading.Thread):
         self.label_array = np.ctypeslib.as_array(
             self.fe_kwargs["label_array"]).reshape(
             self.data.image.chunk_shape)
+        #: Writer deque to monitor
+        self.writer_dq = writer_dq
         #: Time counter for feature extraction
         self.t_count = 0
         #: Whether debugging is enabled
@@ -86,6 +93,15 @@ class EventExtractorManagerThread(threading.Thread):
         chunks_processed = 0
         frames_processed = 0
         while True:
+            # If the writer_dq starts filling up, then this could lead to
+            # an oom-kill signal. Stall for the writer to prevent this.
+            ldq = len(self.writer_dq)
+            if ldq > 100:
+                stallsec = ldq / 100
+                self.logger.warning(
+                    f"Stalling {stallsec:.1f}s for slow writer")
+                time.sleep(stallsec)
+
             cur_slot = 0
             unavailable_slots = 0
             # Check all slots for segmented labels

dcnum/logic/ctrl.py

@@ -124,21 +124,23 @@ class DCNumJobRunner(threading.Thread):
         self.logger = logging.getLogger(f"dcnum.Runner-{self.pphash[:2]}")
 
         # Sanity checks
-        for os_env in [
-            "OMP_NUM_THREADS",
-            "MKL_NUM_THREADS",
-            "NUMEXPR_NUM_THREADS",
-                "NUMBA_NUM_THREADS"]:
+        for os_env in ["MKL_NUM_THREADS", "NUMBA_NUM_THREADS",
+                       "NUMEXPR_NUM_THREADS", "NUMPY_NUM_THREADS",
+                       "OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS",
+                       "VECLIB_MAXIMUM_THREADS"]:
             # You should disable multithreading for all major tools that
             # use dcnum.logic. We don't want multithreading, because dcnum
             # uses linear code and relies on multiprocessing for
             # parallelization. This has to be done before importing numpy
             # or any other library affected. In your scripts, you can use:
             #
-            #    os.environ.setdefault("OMP_NUM_THREADS", "1")
             #    os.environ.setdefault("MKL_NUM_THREADS", "1")
-            #    os.environ.setdefault("NUMEXPR_NUM_THREADS", "1")
             #    os.environ.setdefault("NUMBA_NUM_THREADS", "1")
+            #    os.environ.setdefault("NUMEXPR_NUM_THREADS", "1")
+            #    os.environ.setdefault("NUMPY_NUM_THREADS", "1")
+            #    os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
+            #    os.environ.setdefault("OMP_NUM_THREADS", "1")
+            #    os.environ.setdefault("VECLIB_MAXIMUM_THREADS", "1")
             #
             val_act = os.environ.get(os_env)
             if val_act != "1":
@@ -515,7 +517,7 @@ class DCNumJobRunner(threading.Thread):
             num_segmenters = 1
         num_extractors = max(1, num_extractors)
         num_segmenters = max(1, num_segmenters)
-        self.job["segmenter_kwargs"]["num_workers"] = num_segmenters
+        self.job.kwargs["segmenter_kwargs"]["num_workers"] = num_segmenters
 
         slot_chunks = mp_spawn.Array("i", num_slots)
         slot_states = mp_spawn.Array("u", num_slots)
@@ -546,6 +548,7 @@ class DCNumJobRunner(threading.Thread):
             fe_kwargs=fe_kwargs,
             num_workers=num_extractors,
             labels_list=thr_segm.labels_list,
+            writer_dq=writer_dq,
             debug=self.job["debug"])
         thr_feat.start()
 

dcnum/write/deque_writer_thread.py

@@ -41,11 +41,13 @@ class DequeWriterThread(threading.Thread):
 
     def run(self):
         while True:
+            ldq = len(self.dq)
             if self.must_stop_loop:
                 break
-            elif len(self.dq):
-                feat, data = self.dq.popleft()
-                self.writer.store_feature_chunk(feat=feat, data=data)
+            elif ldq:
+                for _ in range(ldq):
+                    feat, data = self.dq.popleft()
+                    self.writer.store_feature_chunk(feat=feat, data=data)
             elif self.may_stop_loop:
                 break
             else:

dcnum/write/writer.py

@@ -35,7 +35,7 @@ class HDF5Writer:
 
     @staticmethod
     def get_best_nd_chunks(item_shape, feat_dtype=np.float64):
-        """Return best chunks for image data
+        """Return best chunks for HDF5 datasets
 
         Chunking has performance implications. It’s recommended to keep the
         total size of dataset chunks between 10 KiB and 1 MiB. This number
@@ -44,6 +44,7 @@ class HDF5Writer:
         """
         # set image feature chunk size to approximately 1MiB
         num_bytes = 1024 ** 2
+        # Note that `np.prod(()) == 1`
         event_size = np.prod(item_shape) * np.dtype(feat_dtype).itemsize
         chunk_size = num_bytes / event_size
         # Set minimum chunk size to 10 so that we can have at least some
@@ -53,12 +54,11 @@ class HDF5Writer:
 
     def require_feature(self, feat, item_shape, feat_dtype, ds_kwds=None):
         """Create a new feature in the "events" group"""
-
-        if ds_kwds is None:
-            ds_kwds = {}
-        for key in self.ds_kwds:
-            ds_kwds.setdefault(key, self.ds_kwds[key])
         if feat not in self.events:
+            if ds_kwds is None:
+                ds_kwds = {}
+            for key in self.ds_kwds:
+                ds_kwds.setdefault(key, self.ds_kwds[key])
             dset = self.events.create_dataset(
                 feat,
                 shape=tuple([0] + list(item_shape)),

{dcnum-0.16.6.dist-info → dcnum-0.16.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.16.6
+Version: 0.16.8
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.16.6.dist-info → dcnum-0.16.8.dist-info}/RECORD

@@ -1,7 +1,7 @@
 dcnum/__init__.py,sha256=hcawIKS7utYiOyVhOAX9t7K3xYzP1b9862VV0b6qSrQ,74
-dcnum/_version.py,sha256=h39u2Mo-S3RRIMlrfW8TU54mTZkmYuu69tEf5q1_V18,413
+dcnum/_version.py,sha256=9RDap9Epg0PWfAHpF1CbM0Nbp6dRrQKcwV8RmgVyzJs,413
 dcnum/feat/__init__.py,sha256=JqlgzOgDJhoTk8WVYcIiKTWq9EAM16_jGivzOtN6JGo,325
-dcnum/feat/event_extractor_manager_thread.py,sha256=re0McYaWv6h-unLj2B0OhfQtWs-prqIazC8vbDUDQ30,6480
+dcnum/feat/event_extractor_manager_thread.py,sha256=Ocid_t1awH6pOmurCmKYkC51XsXB0-DoN3fzjFDgE4c,7129
 dcnum/feat/gate.py,sha256=ZTV2tkNqJBGmwemZrI0IAaPXTLgWO_5ZVxnpqr2G1cI,7464
 dcnum/feat/queue_event_extractor.py,sha256=eacmcM_uHHW6GtqCCtDCZib9423TGvuqgxGKAxv0tgc,15686
 dcnum/feat/feat_background/__init__.py,sha256=OTmMuazHNaSrZb2XW4cnJ6PlgJLbKrPbaidpEixYa0A,341
@@ -19,7 +19,7 @@ dcnum/feat/feat_texture/__init__.py,sha256=6StM9S540UVtdFFR3bHa7nfCTomeVdoo7Uy9C
 dcnum/feat/feat_texture/common.py,sha256=COXHpXS-7DMouGu3WF83I76L02Sr7P9re4lxajh6g0E,439
 dcnum/feat/feat_texture/tex_all.py,sha256=eGjjNfPpfZw7FA_VNFCIMiU38KD0qcGbxLciYy-tCiA,4097
 dcnum/logic/__init__.py,sha256=7J3GrwJInNQbrLk61HRIV7X7p69TAIbMYpR34hh6u14,177
-dcnum/logic/ctrl.py,sha256=a_qroJPyr-UV2M8EZTnYKP4kGnMETvFjBgVyShhVkuw,26692
+dcnum/logic/ctrl.py,sha256=7m1HL_kO62d8Kt_o4gX3bhxbI4pwOhv3HWHRmbCaMp0,27022
 dcnum/logic/job.py,sha256=M0Q-Rfcm-zkTXTQc79W6YSNUjUlgmRPG0Ikbdn1aOpY,4608
 dcnum/logic/json_encoder.py,sha256=dy44ArmdnxpUfxxONmKdIv-fde3aTXPjZDN0HPATaxs,467
 dcnum/meta/__init__.py,sha256=cQT_HN5yDKzMnZM8CUyNmeA68OhE3ENO_rvFmgDj95c,40
@@ -35,11 +35,11 @@ dcnum/segm/segmenter_cpu.py,sha256=tCY105rVr9_0RIq2618qnF1ueHRj7UtuK_nUBoAg-nY,1
 dcnum/segm/segmenter_gpu.py,sha256=tL2X5BN0jKmhC7wgfG0hygd-6UpG1ZCVuKe5OP1qde0,2133
 dcnum/segm/segmenter_manager_thread.py,sha256=2znDaKedSueomcU1pbHtFmVcGoHzp--sf494VgJF_Tk,5342
 dcnum/write/__init__.py,sha256=Cpn3LqL18hh8OScUnGp_AnNfpWPpKW-oAJZH6ot7aRA,241
-dcnum/write/deque_writer_thread.py,sha256=R4x3p-HZUls3upCBX3vV1VqSdSmaiHdrAswMJj_tVpk,1643
+dcnum/write/deque_writer_thread.py,sha256=KpJ6po8JPlM696MITN-bhNnWQcy9E-qlhg9g-uzoPZg,1710
 dcnum/write/queue_collector_thread.py,sha256=YQ6pvKNmCDf1C6HVx6gOA-q-FBoI6nkhOo-tAVYnyag,11906
-dcnum/write/writer.py,sha256=Hr37OSDJGUpJJ4OufJHYYBanE26GiNwUPOMAt-5Yc2Y,10478
-dcnum-0.16.6.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
-dcnum-0.16.6.dist-info/METADATA,sha256=6f6l1RRqcNyxACqTQpg494d_Oxozd14iC29eO60-WfU,2194
-dcnum-0.16.6.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-dcnum-0.16.6.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
-dcnum-0.16.6.dist-info/RECORD,,
+dcnum/write/writer.py,sha256=QGYNda102f2_12YWXu5WEBEQaTXhNnuQ20g-Dej-cek,10535
+dcnum-0.16.8.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
+dcnum-0.16.8.dist-info/METADATA,sha256=OwayeBlT5wdvffELr7znBmnKvlS52I7sdeP8bOB10lo,2194
+dcnum-0.16.8.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+dcnum-0.16.8.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
+dcnum-0.16.8.dist-info/RECORD,,