dcnum 0.23.1__py3-none-any.whl → 0.23.3__py3-none-any.whl

dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.23.1'
-__version_tuple__ = version_tuple = (0, 23, 1)
+__version__ = version = '0.23.3'
+__version_tuple__ = version_tuple = (0, 23, 3)

dcnum/feat/event_extractor_manager_thread.py

@@ -96,12 +96,13 @@ class EventExtractorManagerThread(threading.Thread):
             # If the writer_dq starts filling up, then this could lead to
             # an oom-kill signal. Stall for the writer to prevent this.
             if (ldq := len(self.writer_dq)) > 1000:
-                time.sleep(1)
-                ldq2 = len(self.writer_dq)
-                stall_time = max(0., (ldq2 - 200) / ((ldq - ldq2) or 1))
-                time.sleep(stall_time)
+                stalled_sec = 0.
+                for ii in range(60):
+                    if len(self.writer_dq) > 200:
+                        time.sleep(.5)
+                        stalled_sec += .5
                 self.logger.warning(
-                    f"Stalled {stall_time + 1:.1f}s for slow writer "
+                    f"Stalled {stalled_sec:.1f}s due to slow writer "
                     f"({ldq} chunks queued)")
 
             unavailable_slots = 0

dcnum/logic/ctrl.py

@@ -1,6 +1,7 @@
 import collections
 import datetime
 import hashlib
+import importlib
 import json
 import logging
 from logging.handlers import QueueListener
@@ -33,6 +34,7 @@ from ..write import (
 from .job import DCNumPipelineJob
 from .json_encoder import ExtendedJSONEncoder
 
+
 # Force using "spawn" method for multiprocessing, because we are using
 # queues and threads and would end up with race conditions otherwise.
 mp_spawn = mp.get_context("spawn")
@@ -430,6 +432,16 @@ class DCNumJobRunner(threading.Thread):
                                  "build": ", ".join(platform.python_build()),
                                  "implementation":
                                      platform.python_implementation(),
+                                 "libraries": get_library_versions_dict([
+                                     "cv2",
+                                     "h5py",
+                                     "mahotas",
+                                     "numba",
+                                     "numpy",
+                                     "scipy",
+                                     "skimage",
+                                     "torch",
+                                 ]),
                                  "version": platform.python_version(),
                                  },
                              "system": {
@@ -551,8 +563,8 @@ class DCNumJobRunner(threading.Thread):
             # 3. image features from the input file
             [self.draw.h5, ["image", "image_bg", "bg_off"], "optional"],
         ]
-        with h5py.File(self.path_temp_out, "a") as hout:
-            hw = HDF5Writer(hout)
+        with HDF5Writer(self.path_temp_out) as hw:
+            hout = hw.h5
             # First, we have to determine the basin mapping from input to
             # output. This information is stored by the QueueCollectorThread
             # in the "basinmap0" feature, ready to be used by us.
@@ -565,7 +577,7 @@ class DCNumJobRunner(threading.Thread):
                 # mapping of the input file was set to slice(1, 100), then the
                 # first image would not be there, and we would have
                 # [1, 1, 1, ...].
-                idx_um = hout["events/index_unmapped"]
+                idx_um = hout["events/index_unmapped"][:]
 
                 # If we want to convert this to an actual basinmap feature,
                 # then we have to convert those indices to indices that map
@@ -780,6 +792,19 @@ class DCNumJobRunner(threading.Thread):
         self.logger.info("Finished segmentation and feature extraction")
 
 
+def get_library_versions_dict(library_name_list):
+    version_dict = {}
+    for library_name in library_name_list:
+        try:
+            lib = importlib.import_module(library_name)
+        except BaseException:
+            version = None
+        else:
+            version = lib.__version__
+        version_dict[library_name] = version
+    return version_dict
+
+
 def join_thread_helper(thr, timeout, retries, logger, name):
     for _ in range(retries):
         thr.join(timeout=timeout)

dcnum/logic/job.py

@@ -182,3 +182,25 @@ class DCNumPipelineJob:
         if len(ret) == 1:
             ret = ret[0]
         return ret
+
+    def validate(self):
+        """Make sure the pipeline will run given the job kwargs
+
+        Returns
+        -------
+        True:
+            for testing convenience
+
+        Raises
+        ------
+        dcnum.segm.SegmenterNotApplicableError:
+            the segmenter is incompatible with the input path
+        """
+        # Check segmenter applicability applicability
+        seg_cls = get_available_segmenters()[self.kwargs["segmenter_code"]]
+        with HDF5Data(self.kwargs["path_in"]) as hd:
+            seg_cls.validate_applicability(
+                segmenter_kwargs=self.kwargs["segmenter_kwargs"],
+                logs=hd.logs,
+                meta=hd.meta)
+        return True

dcnum/read/hdf5_data.py

@@ -205,7 +205,9 @@ class HDF5Data:
                     bn_data = "\n".join(
                         [s.decode() for s in h5["basins"][bnkey][:].tolist()])
                     bn_dict = json.loads(bn_data)
-                    basins.append(bn_dict)
+                    if bn_dict["type"] == "file":
+                        # we only support file-based basins
+                        basins.append(bn_dict)
                 self.basins = sorted(basins, key=lambda x: x["name"])
 
         if state["pixel_size"] is not None:

dcnum/segm/segm_torch/__init__.py

@@ -1,16 +1,20 @@
 import importlib
+import warnings
 
 try:
     torch = importlib.import_module("torch")
     req_maj = 2
-    req_min = 3
+    req_min = 2
     ver_tuple = torch.__version__.split(".")
     act_maj = int(ver_tuple[0])
     act_min = int(ver_tuple[1])
     if act_maj < req_maj or (act_maj == req_maj and act_min < req_min):
-        raise ValueError(f"Your PyTorch version {act_maj}.{act_min} is not "
-                         f"supported, please update to at least "
-                         f"{req_maj}.{req_min}")
+        warnings.warn(f"Your PyTorch version {act_maj}.{act_min} is "
+                      f"not supported, please update to at least "
+                      f"{req_maj}.{req_min} to use dcnum's PyTorch"
+                      f"segmenters")
+        raise ImportError(
+            f"Could not find PyTorch {req_maj}.{req_min}")
 except ImportError:
     pass
 else:

dcnum/write/writer.py

@@ -48,7 +48,13 @@ class HDF5Writer:
             self.h5 = obj
             self.h5_owned = False
         else:
-            self.h5 = h5py.File(obj, mode=mode, libver="latest")
+            self.h5 = h5py.File(obj,
+                                mode=mode,
+                                libver="latest",
+                                # Set chunk cache size to 3 MiB for each
+                                # dataset to allow partial writes.
+                                rdcc_nbytes=3145728,
+                                )
             self.h5_owned = True
         self.events = self.h5.require_group("events")
         ds_kwds = set_default_filter_kwargs(ds_kwds)
@@ -323,8 +329,6 @@ def copy_features(h5_src: h5py.File,
     """
     ei = h5_src["events"]
     eo = h5_dst.require_group("events")
-    # This is the size of the output dataset
-    size = h5_dst.attrs["experiment:event count"]
     hw = HDF5Writer(h5_dst)
     for feat in features:
         if feat in eo:
@@ -341,20 +345,28 @@ def copy_features(h5_src: h5py.File,
                           dst_name=feat.encode(),
                           )
         else:
-            # Perform mapping and store the features in chunks to keep
-            # memory usage down.
+            # We have to perform mapping.
+            # Since h5py is very slow at indexing with arrays,
+            # we instead read the data in chunks from the input file,
+            # and perform the mapping afterward using the numpy arrays.
             dsi = ei[feat]
             chunk_size = hw.get_best_nd_chunks(dsi[0].shape, dsi.dtype)[0]
+            size_in = dsi.shape[0]
             start = 0
-            while start < size:
-                chunk_idx = mapping[start:start + chunk_size]
-                # h5py only supports indexing in increasing order
-                chunk_unique, order = np.unique(chunk_idx, return_inverse=True)
-                data_unique = dsi[chunk_unique]
-                data = data_unique[order]
+            while start < size_in:
+                # Get a big chunk of data
+                big_chunk = 10 * chunk_size
+                stop = start + big_chunk
+                data_in = dsi[start:stop]
+                # Determine the indices that we need from that chunk.
+                mapping_idx = (start <= mapping) * (mapping < stop)
+                mapping_chunk = mapping[mapping_idx] - start
+                data = data_in[mapping_chunk]
+                # Note that HDF5 does its own caching, properly handling
+                # partial chunk writes.
                 hw.store_feature_chunk(feat, data)
                 # increment start
-                start += chunk_size
+                start = stop
 
 
 def copy_metadata(h5_src: h5py.File,

{dcnum-0.23.1.dist-info → dcnum-0.23.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.23.1
+Version: 0.23.3
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -26,7 +26,7 @@ Requires-Dist: opencv-python-headless
 Requires-Dist: scikit-image
 Requires-Dist: scipy >=1.8.0
 Provides-Extra: torch
-Requires-Dist: torch >=2.3 ; extra == 'torch'
+Requires-Dist: torch >=2.2 ; extra == 'torch'
 
 |dcnum|
 =======

{dcnum-0.23.1.dist-info → dcnum-0.23.3.dist-info}/RECORD

@@ -1,7 +1,7 @@
 dcnum/__init__.py,sha256=hcawIKS7utYiOyVhOAX9t7K3xYzP1b9862VV0b6qSrQ,74
-dcnum/_version.py,sha256=FvSfCBPqaPzqv_nSHIaBJNolXopVkF1cRxifqhCXk4g,413
+dcnum/_version.py,sha256=9qJUsi4XZXN5b8lZ0o4HLiLegFWN6GNFX7DcJU4ehOE,413
 dcnum/feat/__init__.py,sha256=jUJYWTD3VIoDNKrmryXbjHb1rGwYtK4b7VPWihYgUoo,325
-dcnum/feat/event_extractor_manager_thread.py,sha256=5HdCQCywyQ5QC56AMjSqCroqif9oOFyiSFWCe07JojM,7820
+dcnum/feat/event_extractor_manager_thread.py,sha256=FAxSyRfaNAuBWNplxHngp5h-44s0qIP24XX_oETdfMk,7836
 dcnum/feat/gate.py,sha256=Yhxq80JoRMmQzBxl35C8NT91c9QcmQa-EIKLuxK6WvE,7221
 dcnum/feat/queue_event_extractor.py,sha256=0ncTQleT1sfc98zYkFuZWxU-akecfTrW6-OOU3z-d8o,15698
 dcnum/feat/feat_background/__init__.py,sha256=OTmMuazHNaSrZb2XW4cnJ6PlgJLbKrPbaidpEixYa0A,341
@@ -20,8 +20,8 @@ dcnum/feat/feat_texture/__init__.py,sha256=6StM9S540UVtdFFR3bHa7nfCTomeVdoo7Uy9C
 dcnum/feat/feat_texture/common.py,sha256=COXHpXS-7DMouGu3WF83I76L02Sr7P9re4lxajh6g0E,439
 dcnum/feat/feat_texture/tex_all.py,sha256=_5H3sXYRN0Uq2eUHn3XUyEHkU_tncEqbqJTC-HZcnGY,5198
 dcnum/logic/__init__.py,sha256=7J3GrwJInNQbrLk61HRIV7X7p69TAIbMYpR34hh6u14,177
-dcnum/logic/ctrl.py,sha256=FyVlizHOIaIGMqINvM-outPywAQU0-5NM7t1dEDml4c,35332
-dcnum/logic/job.py,sha256=H1uDZ1nnNHNWWCe6mS8OWB0Uxc6XUKLISx5xExeplZY,7015
+dcnum/logic/ctrl.py,sha256=sgn1gxctsiPjzD4hFQpzcPGeHP08yjMhWuwG0ocijeY,36182
+dcnum/logic/job.py,sha256=9BN2WjYqjjJuLnfNZAtQ2Nn47Glo2jVrivDodGJoqlQ,7713
 dcnum/logic/json_encoder.py,sha256=cxMnqisbKEVf-rVcw6rK2BBAb6iz_hKFaGl81kK36lQ,571
 dcnum/meta/__init__.py,sha256=AVqRgyKXO1orKnE305h88IBvoZ1oz6X11HN1WP5nGvg,60
 dcnum/meta/paths.py,sha256=J_ikeHzd7gEeRgAKjuayz3x6q4h1fOiDadM-ZxhAGm4,1053
@@ -29,7 +29,7 @@ dcnum/meta/ppid.py,sha256=OD79NrZ8waC3julwdH8NjneUuXqSRSHsUDpKzT5pdyU,8432
 dcnum/read/__init__.py,sha256=ksLdV8EkOU3EPje8teCOSehcUeGAZfg9TQ5ltuEUgls,216
 dcnum/read/cache.py,sha256=lisrGG7AyvVitf0h92wh5FvYCsxa0pWyGcAyYwGP-LQ,6471
 dcnum/read/const.py,sha256=GG9iyXDtEldvJYOBnhZjlimzIeBMAt4bSr2-xn2gzzc,464
-dcnum/read/hdf5_data.py,sha256=Yyq02UTILc5ZgIQXpR9Y0wuX2WT8s0g23PraI7KxmJY,23489
+dcnum/read/hdf5_data.py,sha256=g1kcq9nK2eA063qlAWyvYq1UuRCxdCmWqjK8VTxE8Hk,23603
 dcnum/read/mapped.py,sha256=UryArlrIsHxjOyimBL2Nooi3r73zuGtnGdqdxa6PK_g,3076
 dcnum/segm/__init__.py,sha256=9cLEAd3JWE8IGqDHV-eSDIYOGBfOepd8OcebtNs8Omk,309
 dcnum/segm/segm_thresh.py,sha256=iVhvIhzO0Gw0t3rXOgH71rOI0CNjJJQq4Gg6BulUhK8,948
@@ -37,7 +37,7 @@ dcnum/segm/segmenter.py,sha256=FWLFDBR-x_85ku2rObA2F-QBrM4IUaUL-YHChLagVvM,14902
 dcnum/segm/segmenter_manager_thread.py,sha256=frM0sMxC7f7TQiFjmpRxuwG2kUBFpW1inV8dtpADHiI,5924
 dcnum/segm/segmenter_mpo.py,sha256=o6mQlITHgEWvQt9v6oCWwAcZUvxE7MOeLE9DFManzpY,13757
 dcnum/segm/segmenter_sto.py,sha256=e6MtN_RWusA0wTExV-FLGpDXNJs1CbSyXcSdWUPBMvM,3959
-dcnum/segm/segm_torch/__init__.py,sha256=re9jVLYvV1GgC7J5vx2LHKeFYVZPpiwubecAV9f_2kA,670
+dcnum/segm/segm_torch/__init__.py,sha256=DtUqJTbj7ybrTbXlwHq1Y4SCzi22rMW9Cus6wX-iU-A,822
 dcnum/segm/segm_torch/segm_torch_base.py,sha256=G9AhVyD6LkAmk0tkbYnJUSpvcj3_HYf0uqfILZQsyus,4479
 dcnum/segm/segm_torch/segm_torch_mpo.py,sha256=N01dVXai_4eIGfHJrPjg5C2Bkyq1TOeXeJhw3YbGidw,2504
 dcnum/segm/segm_torch/segm_torch_sto.py,sha256=PTOJrP_FkaxZZul8lM4VA2HL3KyxrheDDWWdJbmJdiw,3393
@@ -47,9 +47,9 @@ dcnum/segm/segm_torch/torch_preproc.py,sha256=kjabu76paw23kO7RP7Ik6IY60Kk1VBAHKB
 dcnum/write/__init__.py,sha256=QvWHeZmjHI18i-YlGYuzN3i7dVWY9UCReKchrJ-gif0,260
 dcnum/write/deque_writer_thread.py,sha256=ao7F1yrVKyufgC4rC0Y2_Vt7snuT6KpI7W2qVxcjdhk,1994
 dcnum/write/queue_collector_thread.py,sha256=d_WfdsZdFnFsiAY0zVMwUlA4juIMeiWYmE_-rezBQCE,11734
-dcnum/write/writer.py,sha256=e6J8YVqhS7kzkpPIMoDMokJpqSy1WWNdOrwaJof1oVc,15601
-dcnum-0.23.1.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
-dcnum-0.23.1.dist-info/METADATA,sha256=NL79bLTjTqm5R4HW0TNcVZ9YXqbxH7Xcfl6xuk-hfw4,2280
-dcnum-0.23.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-dcnum-0.23.1.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
-dcnum-0.23.1.dist-info/RECORD,,
+dcnum/write/writer.py,sha256=H0XoX6lOi1D1sa-ea5SLVftIQY-gdSIVq4vihYv0ODo,16200
+dcnum-0.23.3.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
+dcnum-0.23.3.dist-info/METADATA,sha256=bqOjeHyxCqI4R5leSsFWtw-Ff0ywxQtsA_GuGB1hVNE,2280
+dcnum-0.23.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+dcnum-0.23.3.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
+dcnum-0.23.3.dist-info/RECORD,,