dcnum 0.23.2__tar.gz → 0.23.3__tar.gz

{dcnum-0.23.2 → dcnum-0.23.3}/CHANGELOG

@@ -1,3 +1,7 @@
+0.23.3
+ - fix: ignore non-file-type-like basins
+ - fix: workaround for slow reading from HDF5 (don't use index arrays)
+ - fix: avoid excessive stalling when writer is slow
 0.23.2
  - enh: add DCNumPipelineJob.validate method
  - enh: list Python libraries used in job log

{dcnum-0.23.2 → dcnum-0.23.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.23.2
+Version: 0.23.3
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.23.2 → dcnum-0.23.3}/src/dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.23.2'
-__version_tuple__ = version_tuple = (0, 23, 2)
+__version__ = version = '0.23.3'
+__version_tuple__ = version_tuple = (0, 23, 3)

{dcnum-0.23.2 → dcnum-0.23.3}/src/dcnum/feat/event_extractor_manager_thread.py

@@ -96,12 +96,13 @@ class EventExtractorManagerThread(threading.Thread):
             # If the writer_dq starts filling up, then this could lead to
             # an oom-kill signal. Stall for the writer to prevent this.
             if (ldq := len(self.writer_dq)) > 1000:
-                time.sleep(1)
-                ldq2 = len(self.writer_dq)
-                stall_time = max(0., (ldq2 - 200) / ((ldq - ldq2) or 1))
-                time.sleep(stall_time)
+                stalled_sec = 0.
+                for ii in range(60):
+                    if len(self.writer_dq) > 200:
+                        time.sleep(.5)
+                        stalled_sec += .5
                 self.logger.warning(
-                    f"Stalled {stall_time + 1:.1f}s for slow writer "
+                    f"Stalled {stalled_sec:.1f}s due to slow writer "
                     f"({ldq} chunks queued)")
 
             unavailable_slots = 0

{dcnum-0.23.2 → dcnum-0.23.3}/src/dcnum/logic/ctrl.py

@@ -34,6 +34,7 @@ from ..write import (
 from .job import DCNumPipelineJob
 from .json_encoder import ExtendedJSONEncoder
 
+
 # Force using "spawn" method for multiprocessing, because we are using
 # queues and threads and would end up with race conditions otherwise.
 mp_spawn = mp.get_context("spawn")
@@ -562,8 +563,8 @@ class DCNumJobRunner(threading.Thread):
             # 3. image features from the input file
             [self.draw.h5, ["image", "image_bg", "bg_off"], "optional"],
         ]
-        with h5py.File(self.path_temp_out, "a") as hout:
-            hw = HDF5Writer(hout)
+        with HDF5Writer(self.path_temp_out) as hw:
+            hout = hw.h5
             # First, we have to determine the basin mapping from input to
             # output. This information is stored by the QueueCollectorThread
             # in the "basinmap0" feature, ready to be used by us.
@@ -576,7 +577,7 @@ class DCNumJobRunner(threading.Thread):
                 # mapping of the input file was set to slice(1, 100), then the
                 # first image would not be there, and we would have
                 # [1, 1, 1, ...].
-                idx_um = hout["events/index_unmapped"]
+                idx_um = hout["events/index_unmapped"][:]
 
                 # If we want to convert this to an actual basinmap feature,
                 # then we have to convert those indices to indices that map

{dcnum-0.23.2 → dcnum-0.23.3}/src/dcnum/read/hdf5_data.py

@@ -205,7 +205,9 @@ class HDF5Data:
                     bn_data = "\n".join(
                         [s.decode() for s in h5["basins"][bnkey][:].tolist()])
                     bn_dict = json.loads(bn_data)
-                    basins.append(bn_dict)
+                    if bn_dict["type"] == "file":
+                        # we only support file-based basins
+                        basins.append(bn_dict)
                 self.basins = sorted(basins, key=lambda x: x["name"])
 
         if state["pixel_size"] is not None:

{dcnum-0.23.2 → dcnum-0.23.3}/src/dcnum/write/writer.py

@@ -48,7 +48,13 @@ class HDF5Writer:
             self.h5 = obj
             self.h5_owned = False
         else:
-            self.h5 = h5py.File(obj, mode=mode, libver="latest")
+            self.h5 = h5py.File(obj,
+                                mode=mode,
+                                libver="latest",
+                                # Set chunk cache size to 3 MiB for each
+                                # dataset to allow partial writes.
+                                rdcc_nbytes=3145728,
+                                )
             self.h5_owned = True
         self.events = self.h5.require_group("events")
         ds_kwds = set_default_filter_kwargs(ds_kwds)
@@ -323,8 +329,6 @@ def copy_features(h5_src: h5py.File,
     """
     ei = h5_src["events"]
     eo = h5_dst.require_group("events")
-    # This is the size of the output dataset
-    size = h5_dst.attrs["experiment:event count"]
     hw = HDF5Writer(h5_dst)
     for feat in features:
         if feat in eo:
@@ -341,20 +345,28 @@ def copy_features(h5_src: h5py.File,
                           dst_name=feat.encode(),
                           )
         else:
-            # Perform mapping and store the features in chunks to keep
-            # memory usage down.
+            # We have to perform mapping.
+            # Since h5py is very slow at indexing with arrays,
+            # we instead read the data in chunks from the input file,
+            # and perform the mapping afterward using the numpy arrays.
             dsi = ei[feat]
             chunk_size = hw.get_best_nd_chunks(dsi[0].shape, dsi.dtype)[0]
+            size_in = dsi.shape[0]
             start = 0
-            while start < size:
-                chunk_idx = mapping[start:start + chunk_size]
-                # h5py only supports indexing in increasing order
-                chunk_unique, order = np.unique(chunk_idx, return_inverse=True)
-                data_unique = dsi[chunk_unique]
-                data = data_unique[order]
+            while start < size_in:
+                # Get a big chunk of data
+                big_chunk = 10 * chunk_size
+                stop = start + big_chunk
+                data_in = dsi[start:stop]
+                # Determine the indices that we need from that chunk.
+                mapping_idx = (start <= mapping) * (mapping < stop)
+                mapping_chunk = mapping[mapping_idx] - start
+                data = data_in[mapping_chunk]
+                # Note that HDF5 does its own caching, properly handling
+                # partial chunk writes.
                 hw.store_feature_chunk(feat, data)
                 # increment start
-                start += chunk_size
+                start = stop
 
 
 def copy_metadata(h5_src: h5py.File,

{dcnum-0.23.2 → dcnum-0.23.3}/src/dcnum.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.23.2
+Version: 0.23.3
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.23.2 → dcnum-0.23.3}/tests/test_write_writer.py

@@ -84,6 +84,43 @@ def test_copy_features_error_type_group():
                                 )
 
 
+@pytest.mark.parametrize("samples", [15, 89, 500, 714, 965, 1482])
+def test_copy_features_large_dataset(samples):
+    """
+    Make sure mapping works properly for datasets that are larger
+    than the regular chunk size.
+    """
+    path_orig = retrieve_data(
+        "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+
+    # create large input file
+    path_large = path_orig.with_name("large.rtdc")
+    with write.HDF5Writer(path_large) as hw, read.HDF5Data(path_orig) as hd:
+        write.copy_metadata(h5_src=hd.h5, h5_dst=hw.h5)
+        image = hd.h5["events/image"][:]
+        iterations = (1000 // image.shape[0]) + 1
+        for ii in range(iterations):
+            hw.store_feature_chunk("image", image)
+        size = hw.h5["events/image"].shape[0]
+        hw.h5.attrs["experiment:event count"] = size
+        # 1000 should be big enough for chunk sizes of about 40
+        assert size > 1000, "sanity check"
+
+    # define output mapping
+    mapping = np.sort(np.random.randint(low=0, high=size, size=samples))
+    assert len(mapping) == samples
+
+    # now write to the output file
+    path_out = path_orig.with_name("output.rtdc")
+    with h5py.File(path_large) as hl, h5py.File(path_out, "a") as ho:
+        write.copy_features(h5_src=hl, h5_dst=ho,
+                            features=["image"], mapping=mapping)
+
+    # make sure this worked
+    with h5py.File(path_large) as hl, h5py.File(path_out) as ho:
+        assert np.all(hl["events/image"][:][mapping] == ho["events/image"][:])
+
+
 def test_copy_metadata_empty_log_variable_length_string():
     path = retrieve_data(
         "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")