dcnum 0.16.6__tar.gz → 0.16.8__tar.gz

{dcnum-0.16.6 → dcnum-0.16.8}/CHANGELOG

@@ -1,3 +1,11 @@
+0.16.8
+ - fix: correctly set number of workers for CPUSegmenter
+ - enh: update list of environment variables that should be set to
+   disable multithreading in subprocesses
+0.16.7
+ - fix: if the writer dequeue fills up, stall the feature extractor
+ - enh: optimize DequeWriterThread loop
+ - enh: minor optimization in HDF5Writer.require_feature
 0.16.6
  - fix: correctly handle mask images with no background on border
  - fix: enforce user-defined features in concatenated_hdf5_data
@@ -19,7 +27,7 @@
  - enh: define valid DCNumJobRunner state
  - enh: more robust computation of progress
  - enh: use HDF5Data when loading input data for background computation
- - enh: automatically split segmenters and axtractors equally
+ - enh: automatically split segmenters and extractors equally
  - ref: reduce default image cache size from 5 to 2
  - ref: move dataset generation default kwargs to writer submodule
  - ref: warn above 0.5% of discarded events in EventExtractorManagerThread

{dcnum-0.16.6 → dcnum-0.16.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.16.6
+Version: 0.16.8
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.16.6 → dcnum-0.16.8}/src/dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.16.6'
-__version_tuple__ = version_tuple = (0, 16, 6)
+__version__ = version = '0.16.8'
+__version_tuple__ = version_tuple = (0, 16, 8)

{dcnum-0.16.6 → dcnum-0.16.8}/src/dcnum/feat/event_extractor_manager_thread.py

@@ -1,4 +1,5 @@
 """Feature computation: managing event extraction threads"""
+import collections
 import logging
 import multiprocessing as mp
 import threading
@@ -17,6 +18,7 @@ class EventExtractorManagerThread(threading.Thread):
                  labels_list: List,
                  fe_kwargs: Dict,
                  num_workers: int,
+                 writer_dq: collections.deque,
                  debug: bool = False,
                  *args, **kwargs):
         """Manage event extraction threads or precesses
@@ -40,6 +42,9 @@ class EventExtractorManagerThread(threading.Thread):
             :func:`.EventExtractor.get_init_kwargs` for more information.
         num_workers:
             Number of child threads or worker processes to use.
+        writer_dq:
+            The queue the writer uses. We monitor this queue. If it
+            fills up, we take a break.
         debug:
             Whether to run in debugging mode which means more log
             messages and only one thread (`num_workers` has no effect).
@@ -66,6 +71,8 @@ class EventExtractorManagerThread(threading.Thread):
         self.label_array = np.ctypeslib.as_array(
             self.fe_kwargs["label_array"]).reshape(
             self.data.image.chunk_shape)
+        #: Writer deque to monitor
+        self.writer_dq = writer_dq
         #: Time counter for feature extraction
         self.t_count = 0
         #: Whether debugging is enabled
@@ -86,6 +93,15 @@ class EventExtractorManagerThread(threading.Thread):
         chunks_processed = 0
         frames_processed = 0
         while True:
+            # If the writer_dq starts filling up, then this could lead to
+            # an oom-kill signal. Stall for the writer to prevent this.
+            ldq = len(self.writer_dq)
+            if ldq > 100:
+                stallsec = ldq / 100
+                self.logger.warning(
+                    f"Stalling {stallsec:.1f}s for slow writer")
+                time.sleep(stallsec)
+
             cur_slot = 0
             unavailable_slots = 0
             # Check all slots for segmented labels

{dcnum-0.16.6 → dcnum-0.16.8}/src/dcnum/logic/ctrl.py

@@ -124,21 +124,23 @@ class DCNumJobRunner(threading.Thread):
         self.logger = logging.getLogger(f"dcnum.Runner-{self.pphash[:2]}")
 
         # Sanity checks
-        for os_env in [
-            "OMP_NUM_THREADS",
-            "MKL_NUM_THREADS",
-            "NUMEXPR_NUM_THREADS",
-                "NUMBA_NUM_THREADS"]:
+        for os_env in ["MKL_NUM_THREADS", "NUMBA_NUM_THREADS",
+                       "NUMEXPR_NUM_THREADS", "NUMPY_NUM_THREADS",
+                       "OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS",
+                       "VECLIB_MAXIMUM_THREADS"]:
             # You should disable multithreading for all major tools that
             # use dcnum.logic. We don't want multithreading, because dcnum
             # uses linear code and relies on multiprocessing for
             # parallelization. This has to be done before importing numpy
             # or any other library affected. In your scripts, you can use:
             #
-            #    os.environ.setdefault("OMP_NUM_THREADS", "1")
             #    os.environ.setdefault("MKL_NUM_THREADS", "1")
-            #    os.environ.setdefault("NUMEXPR_NUM_THREADS", "1")
             #    os.environ.setdefault("NUMBA_NUM_THREADS", "1")
+            #    os.environ.setdefault("NUMEXPR_NUM_THREADS", "1")
+            #    os.environ.setdefault("NUMPY_NUM_THREADS", "1")
+            #    os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
+            #    os.environ.setdefault("OMP_NUM_THREADS", "1")
+            #    os.environ.setdefault("VECLIB_MAXIMUM_THREADS", "1")
             #
             val_act = os.environ.get(os_env)
             if val_act != "1":
@@ -515,7 +517,7 @@ class DCNumJobRunner(threading.Thread):
             num_segmenters = 1
         num_extractors = max(1, num_extractors)
         num_segmenters = max(1, num_segmenters)
-        self.job["segmenter_kwargs"]["num_workers"] = num_segmenters
+        self.job.kwargs["segmenter_kwargs"]["num_workers"] = num_segmenters
 
         slot_chunks = mp_spawn.Array("i", num_slots)
         slot_states = mp_spawn.Array("u", num_slots)
@@ -546,6 +548,7 @@ class DCNumJobRunner(threading.Thread):
             fe_kwargs=fe_kwargs,
             num_workers=num_extractors,
             labels_list=thr_segm.labels_list,
+            writer_dq=writer_dq,
             debug=self.job["debug"])
         thr_feat.start()
 

{dcnum-0.16.6 → dcnum-0.16.8}/src/dcnum/write/deque_writer_thread.py

@@ -41,11 +41,13 @@ class DequeWriterThread(threading.Thread):
 
     def run(self):
         while True:
+            ldq = len(self.dq)
             if self.must_stop_loop:
                 break
-            elif len(self.dq):
-                feat, data = self.dq.popleft()
-                self.writer.store_feature_chunk(feat=feat, data=data)
+            elif ldq:
+                for _ in range(ldq):
+                    feat, data = self.dq.popleft()
+                    self.writer.store_feature_chunk(feat=feat, data=data)
             elif self.may_stop_loop:
                 break
             else:

{dcnum-0.16.6 → dcnum-0.16.8}/src/dcnum/write/writer.py

@@ -35,7 +35,7 @@ class HDF5Writer:
 
     @staticmethod
     def get_best_nd_chunks(item_shape, feat_dtype=np.float64):
-        """Return best chunks for image data
+        """Return best chunks for HDF5 datasets
 
         Chunking has performance implications. It’s recommended to keep the
         total size of dataset chunks between 10 KiB and 1 MiB. This number
@@ -44,6 +44,7 @@ class HDF5Writer:
         """
         # set image feature chunk size to approximately 1MiB
         num_bytes = 1024 ** 2
+        # Note that `np.prod(()) == 1`
         event_size = np.prod(item_shape) * np.dtype(feat_dtype).itemsize
         chunk_size = num_bytes / event_size
         # Set minimum chunk size to 10 so that we can have at least some
@@ -53,12 +54,11 @@ class HDF5Writer:
 
     def require_feature(self, feat, item_shape, feat_dtype, ds_kwds=None):
         """Create a new feature in the "events" group"""
-
-        if ds_kwds is None:
-            ds_kwds = {}
-        for key in self.ds_kwds:
-            ds_kwds.setdefault(key, self.ds_kwds[key])
         if feat not in self.events:
+            if ds_kwds is None:
+                ds_kwds = {}
+            for key in self.ds_kwds:
+                ds_kwds.setdefault(key, self.ds_kwds[key])
             dset = self.events.create_dataset(
                 feat,
                 shape=tuple([0] + list(item_shape)),

{dcnum-0.16.6 → dcnum-0.16.8}/src/dcnum.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.16.6
+Version: 0.16.8
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.16.6 → dcnum-0.16.8}/tests/test_feat_event_extractor_manager.py

@@ -1,3 +1,4 @@
+import collections
 import logging
 import multiprocessing as mp
 import queue
@@ -48,6 +49,7 @@ def test_event_extractor_manager_thread():
         fe_kwargs=fe_kwargs,
         num_workers=1,
         labels_list=thr_segm.labels_list,
+        writer_dq=collections.deque(),
         debug=True)
     thr_feat.run()
     thr_segm.join()

{dcnum-0.16.6 → dcnum-0.16.8}/tests/test_read_basin.py

@@ -4,7 +4,6 @@ import numpy as np
 from dcnum.write import HDF5Writer, create_with_basins
 from dcnum.read import HDF5Data
 
-
 from helper_methods import retrieve_data
 
 
@@ -184,3 +183,25 @@ def test_basin_scalar_features():
         assert np.allclose(hd["deform"][0], 0.0740563677588885)
         assert np.allclose(hd["area_um"][0], 0.559682)
         assert np.allclose(hd["area_um"][1], 91.193185875)
+
+
+def test_basin_self_reference():
+    """Paths can self-reference in basins, no recursion errors"""
+    h5path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
+
+    # Dataset creation
+    with HDF5Writer(h5path, "a") as hw:
+        # Next, store the basin information in the new dataset
+        hw.store_basin(name="test",
+                       paths=[h5path])
+
+    # Now open the scalar dataset and check whether basins are defined
+    with HDF5Data(h5path) as hd:
+        assert "image" in hd.get_basin_data(0)[1]
+        assert "image" in hd.keys()
+        assert np.median(hd["image"][0]) == 187
+        assert np.median(hd.image[0]) == 187
+        assert np.median(hd.image_corr[0]) == 1
+        assert np.allclose(np.mean(hd["deform"]),
+                           0.23354564471483724,
+                           atol=0, rtol=1e-7)