dcnum 0.11.1__tar.gz → 0.11.2__tar.gz

{dcnum-0.11.1 → dcnum-0.11.2}/CHANGELOG

@@ -1,3 +1,12 @@
+0.11.2
+ - meta: increment pipeline ID (texture feature computation)
+ - fix: HDF5Data was not pickable
+ - fix: HDF5Data did not properly handle tables
+ - enh: add context manager for CPUSegmenter
+ - enh: record and log execution time of segmentation and feature extraction
+ - enh: properly handle border case for computing contour-moments
+ - enh: properly handle empty images/masks in haralick texture features
+ - tests: do not use numba's JIT during testing (coverage)
 0.11.1
  - fix: fix GPUSegmenter labeling
 0.11.0

{dcnum-0.11.1 → dcnum-0.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.11.1
+Version: 0.11.2
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.11.1 → dcnum-0.11.2}/dcnum/_version.py

@@ -1,4 +1,4 @@
 # file generated by setuptools_scm
 # don't change, don't track in version control
-__version__ = version = '0.11.1'
-__version_tuple__ = version_tuple = (0, 11, 1)
+__version__ = version = '0.11.2'
+__version_tuple__ = version_tuple = (0, 11, 2)

{dcnum-0.11.1 → dcnum-0.11.2}/dcnum/feat/event_extractor_manager_thread.py

@@ -65,7 +65,8 @@ class EventExtractorManagerThread(threading.Thread):
         self.label_array = np.ctypeslib.as_array(
             self.fe_kwargs["label_array"]).reshape(
             self.data.image.chunk_shape)
-
+        #: Time counter for feature extraction
+        self.t_count = 0
         #: Whether debugging is enabled
         self.debug = debug
 
@@ -100,6 +101,8 @@ class EventExtractorManagerThread(threading.Thread):
                     unavailable_slots = 0
                     time.sleep(.1)
 
+            t1 = time.monotonic()
+
             # We have a chunk, process it!
             chunk = self.slot_chunks[cur_slot]
             # Populate the labeling array for the workers
@@ -123,8 +126,10 @@ class EventExtractorManagerThread(threading.Thread):
             self.slot_states[cur_slot] = "w"
 
             self.logger.debug(f"Extracted one chunk: {chunk}")
+            self.t_count += time.monotonic() - t1
 
             chunks_processed += 1
+
             if chunks_processed == self.data.image.num_chunks:
                 break
 
@@ -132,3 +137,4 @@ class EventExtractorManagerThread(threading.Thread):
         self.fe_kwargs["finalize_extraction"].value = True
         [w.join() for w in workers]
         self.logger.debug("Finished extraction.")
+        self.logger.info(f"Extraction time: {self.t_count:.1f}s")

{dcnum-0.11.1 → dcnum-0.11.2}/dcnum/feat/feat_moments/mt_legacy.py

@@ -10,22 +10,25 @@ def moments_based_features(mask, pixel_size):
 
     size = mask.shape[0]
 
-    deform = np.zeros(size, dtype=float)
-    size_x = np.zeros(size, dtype=float)
-    size_y = np.zeros(size, dtype=float)
-    pos_x = np.zeros(size, dtype=float)
-    pos_y = np.zeros(size, dtype=float)
-    area_msd = np.zeros(size, dtype=float)
-    area_ratio = np.zeros(size, dtype=float)
-    area_um = np.zeros(size, dtype=float)
-    aspect = np.zeros(size, dtype=float)
-    tilt = np.zeros(size, dtype=float)
-    inert_ratio_cvx = np.zeros(size, dtype=float)
-    inert_ratio_raw = np.zeros(size, dtype=float)
-    inert_ratio_prnc = np.zeros(size, dtype=float)
+    empty = np.full(size, np.nan, dtype=np.float64)
+    deform = np.copy(empty)
+    size_x = np.copy(empty)
+    size_y = np.copy(empty)
+    pos_x = np.copy(empty)
+    pos_y = np.copy(empty)
+    area_msd = np.copy(empty)
+    area_ratio = np.copy(empty)
+    area_um = np.copy(empty)
+    aspect = np.copy(empty)
+    tilt = np.copy(empty)
+    inert_ratio_cvx = np.copy(empty)
+    inert_ratio_raw = np.copy(empty)
+    inert_ratio_prnc = np.copy(empty)
 
     for ii in range(size):
         cont_raw = contour_single_opencv(mask[ii])
+        if len(cont_raw.shape) < 2:
+            continue
         mu_raw = cv2.moments(cont_raw)
 
         # convex hull

{dcnum-0.11.1 → dcnum-0.11.2}/dcnum/feat/feat_texture/tex_all.py

@@ -4,16 +4,16 @@ import numpy as np
 from .common import haralick_names
 
 
-def haralick_texture_features(image, mask, image_bg=None, image_corr=None):
+def haralick_texture_features(
+        mask, image=None, image_bg=None, image_corr=None):
     # make sure we have a boolean array
     mask = np.array(mask, dtype=bool)
     size = mask.shape[0]
 
     # compute features if necessary
-    if image_bg is not None or image_corr is not None:
+    if image_bg is not None and image is not None and image_corr is None:
         # Background-corrected brightness values
-        if image_corr is None:
-            image_corr = np.array(image, dtype=np.int16) - image_bg
+        image_corr = np.array(image, dtype=np.int16) - image_bg
 
     tex_dict = {}
     empty = np.full(size, np.nan, dtype=np.float64)
@@ -29,6 +29,9 @@ def haralick_texture_features(image, mask, image_bg=None, image_corr=None):
         #   -> maximum value should be as small as possible
         # - set pixels outside contour to zero (ignored areas, see mahotas)
         maski = mask[ii]
+        if not np.any(maski):
+            # The mask is empty (nan values)
+            continue
         if image_corr.shape[0] == 1:
             # We have several masks for one image.
             imcoi = image_corr[0]

{dcnum-0.11.1 → dcnum-0.11.2}/dcnum/meta/ppid.py

@@ -8,7 +8,7 @@ import pathlib
 
 #: Increment this string if there are breaking changes that make
 #: previous pipelines unreproducible.
-DCNUM_PPID_GENERATION = "2"
+DCNUM_PPID_GENERATION = "3"
 
 
 def compute_pipeline_hash(bg_id, seg_id, feat_id, gate_id,

{dcnum-0.11.1 → dcnum-0.11.2}/dcnum/read/hdf5_data.py

@@ -31,9 +31,6 @@ class HDF5Data:
         if isinstance(path, h5py.File):
             self.h5 = path
             path = path.filename
-        else:
-            self.h5 = None  # is set in __setstate__
-        self._cache_scalar = {}
         self.__setstate__({"path": path,
                            "pixel_size": pixel_size,
                            "md5_5m": md5_5m,
@@ -69,7 +66,25 @@ class HDF5Data:
             warnings.warn(f"Feature {feat} not cached (possibly slow)")
             return self.h5["events"][feat]
 
+    def __getstate__(self):
+        return {"path": self.path,
+                "pixel_size": self.pixel_size,
+                "md5_5m": self.md5_5m,
+                "meta": self.meta,
+                "logs": self.logs,
+                "tables": self.tables,
+                "image_cache_size": self.image.cache_size
+                }
+
     def __setstate__(self, state):
+        # Make sure these properties exist (we rely on __init__, because
+        # we want this class to be pickable and __init__ is not called by
+        # `pickle.load`.
+        if not hasattr(self, "_cache_scalar"):
+            self._cache_scalar = {}
+        if not hasattr(self, "h5"):
+            self.h5 = None
+
         self.path = state["path"]
 
         self.md5_5m = state["md5_5m"]
@@ -100,7 +115,10 @@ class HDF5Data:
                         alog = [ll.decode("utf") for ll in alog]
                     self.logs[key] = alog
                 for tab in h5.get("tables", []):
-                    self.tables[tab] = h5["tables"][key][:]
+                    tabdict = {}
+                    for tkey in h5["tables"][tab].dtype.fields.keys():
+                        tabdict[tkey] = h5["tables"][tab][tkey]
+                    self.tables[tab] = tabdict
 
         if state["pixel_size"] is not None:
             self.pixel_size = state["pixel_size"]
@@ -137,15 +155,6 @@ class HDF5Data:
 
         self.image_corr = ImageCorrCache(self.image, self.image_bg)
 
-    def __getstate__(self):
-        return {"path": self.path,
-                "pixel_size": self.pixel_size,
-                "md5_5m": self.md5_5m,
-                "meta": self.meta,
-                "logs": self.logs,
-                "tables": self.tables,
-                }
-
     @functools.cache
     def __len__(self):
         return self.h5.attrs["experiment:event count"]

{dcnum-0.11.1 → dcnum-0.11.2}/dcnum/segm/segmenter.py

@@ -179,7 +179,7 @@ class Segmenter(abc.ABC):
 
         if fill_holes:
             # Floodfill only works with uint8 (too small) or int32
-            if not labels.dtype == np.int32:
+            if labels.dtype != np.int32:
                 labels = np.array(labels, dtype=np.int32)
             #
             # from scipy import ndimage

{dcnum-0.11.1 → dcnum-0.11.2}/dcnum/segm/segmenter_cpu.py

@@ -31,6 +31,12 @@ class CPUSegmenter(Segmenter, abc.ABC):
         # Tells the workers to stop
         self.mp_shutdown = mp.Value("i", 0)
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.join_workers()
+
     def __getstate__(self):
         # Copy the object's state from self.__dict__ which contains
         # all our instance attributes. Always use the dict.copy()
@@ -47,7 +53,7 @@ class CPUSegmenter(Segmenter, abc.ABC):
         return state
 
     def __setstate__(self, state):
-        # Restore instance attributes (i.e., filename and lineno).
+        # Restore instance attributes
         self.__dict__.update(state)
 
     @staticmethod

{dcnum-0.11.1 → dcnum-0.11.2}/dcnum/segm/segmenter_manager_thread.py

@@ -71,6 +71,8 @@ class SegmenterManagerThread(threading.Thread):
         self.slot_chunks = slot_chunks
         #: List containing the segmented labels of each slot
         self.labels_list = [None] * len(self.slot_states)
+        #: Time counter for segmentation
+        self.t_count = 0
         #: Whether running in debugging mode
         self.debug = debug
 
@@ -96,6 +98,8 @@ class SegmenterManagerThread(threading.Thread):
                     empty_slots = 0
                     time.sleep(.01)
 
+            t1 = time.monotonic()
+
             # We have a free slot to compute the segmentation
             labels = self.segmenter.segment_chunk(
                 image_data=self.image_data,
@@ -111,7 +115,11 @@ class SegmenterManagerThread(threading.Thread):
             self.slot_states[cur_slot] = "e"
             self.logger.debug(f"Segmented one chunk: {chunk}")
 
+            self.t_count += time.monotonic() - t1
+
         # Cleanup
         if isinstance(self.segmenter, CPUSegmenter):
             # Join the segmentation workers.
             self.segmenter.join_workers()
+
+        self.logger.info(f"Segmentation time: {self.t_count:.1f}s")

{dcnum-0.11.1 → dcnum-0.11.2}/dcnum.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.11.1
+Version: 0.11.2
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.11.1 → dcnum-0.11.2}/dcnum.egg-info/SOURCES.txt

@@ -63,6 +63,7 @@ tests/test_ppid_segm.py
 tests/test_read_concat_hdf5.py
 tests/test_read_hdf5.py
 tests/test_segm_thresh.py
+tests/test_segmenter.py
 tests/test_write_deque_writer_thread.py
 tests/test_write_writer.py
 tests/data/fmt-hdf5_cytoshot_full-features_2023.zip

{dcnum-0.11.1 → dcnum-0.11.2}/tests/conftest.py

@@ -1,4 +1,5 @@
 import atexit
+import os
 import shutil
 import tempfile
 import time
@@ -15,3 +16,5 @@ def pytest_configure(config):
     """
     tempfile.tempdir = TMPDIR
     atexit.register(shutil.rmtree, TMPDIR, ignore_errors=True)
+    # Disable JIT compiler during testing for coverage
+    os.environ.setdefault("NUMBA_DISABLE_JIT", "1")

{dcnum-0.11.1 → dcnum-0.11.2}/tests/test_feat_brightness.py

@@ -31,3 +31,26 @@ def test_basic_brightness():
         # control test
         assert not np.allclose(h5["events"]["bright_perc_10"][:],
                                data["bright_perc_90"])
+
+
+def test_basic_brightness_single_image():
+    # This original file was generated with dcevent for reference.
+    path = retrieve_data(data_path /
+                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    # Make data available
+    with h5py.File(path) as h5:
+        data = feat_brightness.brightness_features(
+            image=h5["events/image"][1][np.newaxis],
+            image_bg=h5["events/image_bg"][1][np.newaxis],
+            mask=h5["events/mask"][1][np.newaxis],
+        )
+
+        assert np.allclose(data["bright_bc_avg"][0],
+                           -43.75497215592681,
+                           atol=0, rtol=1e-10)
+        for feat in feat_brightness.brightness_names:
+            assert np.allclose(h5["events"][feat][1],
+                               data[feat][0]), f"Feature {feat} mismatch!"
+        # control test
+        assert not np.allclose(h5["events"]["bright_perc_10"][1],
+                               data["bright_perc_90"][0])

dcnum-0.11.2/tests/test_feat_haralick.py

@@ -0,0 +1,120 @@
+import pathlib
+
+import h5py
+import numpy as np
+
+from dcnum.feat import feat_texture
+
+from helper_methods import retrieve_data
+
+data_path = pathlib.Path(__file__).parent / "data"
+
+
+def test_basic_haralick():
+    # This original file was generated with dcevent for reference.
+    path = retrieve_data(data_path /
+                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    # Make data available
+    with h5py.File(path) as h5:
+        ret_arr = feat_texture.haralick_texture_features(
+            image=h5["events/image"][:],
+            image_bg=h5["events/image_bg"][:],
+            mask=h5["events/mask"][:],
+        )
+
+        assert np.allclose(ret_arr["tex_asm_avg"][1],
+                           0.001514295993357114,
+                           atol=0, rtol=1e-10)
+        for feat in feat_texture.haralick_names:
+            assert np.allclose(h5["events"][feat],
+                               ret_arr[feat])
+        # control test
+        assert not np.allclose(h5["events"]["tex_asm_avg"],
+                               ret_arr["tex_asm_ptp"])
+
+
+def test_empty_image():
+    masks = np.array([
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 1, 1, 0, 0],
+        [0, 0, 1, 1, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+    ], dtype=bool)[np.newaxis]
+    image_corr = np.zeros(6*6, dtype=np.int16).reshape(1, 6, 6)
+    tex = feat_texture.haralick_texture_features(
+        image_corr=image_corr,
+        mask=masks,
+    )
+    assert np.allclose(tex["tex_con_avg"][0], 0)
+
+
+def test_empty_mask():
+    masks = np.array([
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+    ], dtype=bool)[np.newaxis]
+    image_corr = np.arange(6*6, dtype=np.int16).reshape(1, 6, 6)
+    tex = feat_texture.haralick_texture_features(
+        image_corr=image_corr,
+        mask=masks,
+    )
+    assert np.isnan(tex["tex_con_avg"][0])
+
+
+def test_1d_mask_image():
+    masks = np.array([
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 1, 0, 0, 0],
+        [0, 0, 1, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+    ], dtype=bool)[np.newaxis]
+    image_corr = np.arange(6*6, dtype=np.int16).reshape(1, 6, 6)
+    tex = feat_texture.haralick_texture_features(
+        image_corr=image_corr,
+        mask=masks,
+    )
+    assert np.isnan(tex["tex_con_avg"][0])
+
+
+def test_nd_mask_with_1d_image():
+    mask = np.array([
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 1, 1, 0, 0],
+        [0, 0, 1, 1, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+    ], dtype=bool)
+    masks = np.stack([mask, mask, mask, mask])
+    image_corr = np.arange(6*6, dtype=np.int16).reshape(1, 6, 6)
+    tex = feat_texture.haralick_texture_features(
+        image_corr=image_corr,
+        mask=masks,
+    )
+    assert len(tex["tex_con_avg"]) == 4
+    assert np.allclose(tex["tex_con_avg"][0], 27.75)
+
+
+def test_simple_mask_image():
+    masks = np.array([
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 1, 1, 0, 0],
+        [0, 0, 1, 1, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+    ], dtype=bool)[np.newaxis]
+    image_corr = np.arange(6*6, dtype=np.int16).reshape(1, 6, 6)
+    tex = feat_texture.haralick_texture_features(
+        image_corr=image_corr,
+        mask=masks,
+    )
+    assert np.allclose(tex["tex_con_avg"][0], 27.75)

dcnum-0.11.2/tests/test_feat_moments_based.py

@@ -0,0 +1,108 @@
+import pathlib
+
+import h5py
+import numpy as np
+
+from dcnum.feat import feat_moments
+
+from helper_methods import retrieve_data
+
+data_path = pathlib.Path(__file__).parent / "data"
+
+
+def test_moments_based_features():
+    # This original file was generated with dcevent for reference.
+    path = retrieve_data(data_path /
+                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    feats = [
+        "deform",
+        "size_x",
+        "size_y",
+        "pos_x",
+        "pos_y",
+        "area_msd",
+        "area_ratio",
+        "area_um",
+        "aspect",
+        "tilt",
+        "inert_ratio_cvx",
+        "inert_ratio_raw",
+        "inert_ratio_prnc",
+    ]
+
+    # Make data available
+    with h5py.File(path) as h5:
+        data = feat_moments.moments_based_features(
+            mask=h5["events/mask"][:],
+            pixel_size=0.2645
+        )
+        for feat in feats:
+            if feat.count("inert"):
+                rtol = 2e-5
+                atol = 1e-8
+            else:
+                rtol = 1e-5
+                atol = 1e-8
+            assert np.allclose(h5["events"][feat][:],
+                               data[feat],
+                               rtol=rtol,
+                               atol=atol), f"Feature {feat} mismatch!"
+        # control test
+        assert not np.allclose(h5["events"]["inert_ratio_cvx"][:],
+                               data["tilt"])
+
+
+def test_mask_0d():
+    masks = np.array([
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 1, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+    ], dtype=bool)[np.newaxis]
+    data = feat_moments.moments_based_features(
+                mask=masks,
+                pixel_size=0.2645
+            )
+    assert data["deform"].shape == (1,)
+    assert np.isnan(data["deform"][0])
+    assert np.isnan(data["area_um"][0])
+
+
+def test_mask_1d():
+    masks = np.array([
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 1, 0, 0, 0],
+        [0, 0, 1, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+    ], dtype=bool)[np.newaxis]
+    data = feat_moments.moments_based_features(
+                mask=masks,
+                pixel_size=0.2645
+            )
+    assert data["deform"].shape == (1,)
+    assert np.isnan(data["deform"][0])
+    assert np.isnan(data["area_um"][0])
+
+
+def test_mask_2d():
+    masks = np.array([
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 1, 1, 0, 0],
+        [0, 0, 1, 1, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0],
+    ], dtype=bool)[np.newaxis]
+    data = feat_moments.moments_based_features(
+                mask=masks,
+                pixel_size=0.2645
+            )
+    assert data["deform"].shape == (1,)
+    # This is the deformation of a square (compared to circle)
+    assert np.allclose(data["deform"][0], 0.11377307454724206)
+    # Without moments-based computation, this would be 4*pxsize=0.066125
+    assert np.allclose(data["area_um"][0], 0.06996025)

{dcnum-0.11.1 → dcnum-0.11.2}/tests/test_read_hdf5.py

@@ -134,11 +134,8 @@ def test_pickling_state():
 
     h5d1 = read.HDF5Data(path)
     h5d1.pixel_size = 0.124
-    state = h5d1.__getstate__()
-    pstate = pickle.dumps(state)
-
-    state2 = pickle.loads(pstate)
-    h5d2 = read.HDF5Data(**state2)
+    pstate = pickle.dumps(h5d1)
+    h5d2 = pickle.loads(pstate)
     assert h5d1.md5_5m == h5d2.md5_5m
     assert h5d1.md5_5m == h5d2.md5_5m
     assert h5d1.pixel_size == h5d2.pixel_size
@@ -150,10 +147,40 @@ def test_pickling_state_logs():
         data_path / "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
     h5d1 = read.HDF5Data(path)
     h5d1.pixel_size = 0.124
-    state = h5d1.__getstate__()
-    pstate = pickle.dumps(state)
-
-    state2 = pickle.loads(pstate)
-    h5d2 = read.HDF5Data(**state2)
+    pstate = pickle.dumps(h5d1)
+    h5d2 = pickle.loads(pstate)
+    assert h5d1.logs
     for lk in h5d1.logs:
         assert h5d1.logs[lk] == h5d2.logs[lk]
+
+
+def test_pickling_state_tables():
+    path = retrieve_data(
+        data_path / "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+    # The original file does not contain any tables, so we write
+    # generate a table
+    columns = ["alot", "of", "tables"]
+    ds_dt = np.dtype({'names': columns,
+                      'formats': [float] * len(columns)})
+    tab_data = np.zeros((11, len(columns)))
+    tab_data[:, 0] = np.arange(11)
+    tab_data[:, 1] = 1000
+    tab_data[:, 2] = np.linspace(1, np.sqrt(2), 11)
+    rec_arr = np.rec.array(tab_data, dtype=ds_dt)
+
+    # add table to source file
+    with h5py.File(path, "a") as h5:
+        h5tab = h5.require_group("tables")
+        h5tab.create_dataset(name="sample_table",
+                             data=rec_arr)
+
+    h5d1 = read.HDF5Data(path)
+    h5d1.pixel_size = 0.124
+    pstate = pickle.dumps(h5d1)
+    h5d2 = pickle.loads(pstate)
+    assert h5d1.tables
+    table = h5d1.tables["sample_table"]
+    assert len(table) == 3
+    for lk in table:
+        assert np.allclose(h5d1.tables["sample_table"][lk],
+                           h5d2.tables["sample_table"][lk])

dcnum-0.11.2/tests/test_segm_thresh.py

@@ -0,0 +1,139 @@
+import multiprocessing as mp
+import pathlib
+
+from dcnum import segm
+import h5py
+import numpy as np
+from skimage import morphology
+
+import pytest
+
+from helper_methods import retrieve_data
+
+data_path = pathlib.Path(__file__).parent / "data"
+
+
+def test_segm_thresh_basic():
+    """Basic thresholding segmenter
+
+    The segmenter is equivalent to the old dcevent legacy segmenter with
+    the options legacy:t=-6^bl=0^bi=0^d=1:cle=1^f=1^clo=3
+    (no blur, no binaryops, clear borders, fill holes, closing disk 3).
+    Since in the dcevent pipeline, the data are gated and small objects
+    are removed, we have to do this here manually before comparing mask
+    images.
+    """
+    path = retrieve_data(
+        data_path / "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+
+    # Get all the relevant information
+    with h5py.File(path) as h5:
+        image = h5["events/image"][:]
+        image_bg = h5["events/image_bg"][:]
+        mask = h5["events/mask"][:]
+        frame = h5["events/frame"][:]
+
+    # Concatenate the masks
+    frame_u, indices = np.unique(frame, return_index=True)
+    image_u = image[indices]
+    image_bg_u = image_bg[indices]
+    mask_u = np.zeros_like(image_u, dtype=bool)
+    for ii, fr in enumerate(frame):
+        idx = np.where(frame_u == fr)[0]
+        mask_u[idx] = np.logical_or(mask_u[idx], mask[ii])
+
+    image_u_c = np.array(image_u, dtype=int) - image_bg_u
+
+    sm = segm.segm_thresh.SegmentThresh(thresh=-6,
+                                        kwargs_mask={"closing_disk": 3})
+    for ii in range(len(frame_u)):
+        labels_seg = sm.segment_frame(image_u_c[ii])
+        mask_seg = np.array(labels_seg, dtype=bool)
+        # Remove small objects, because this is not implemented in the
+        # segmenter class as it would be part of gating.
+        mask_seg = morphology.remove_small_objects(mask_seg, min_size=10)
+        assert np.all(mask_seg == mask_u[ii]), f"masks not matching at {ii}"
+
+
+@pytest.mark.parametrize("worker_type", ["thread", "process"])
+def test_segm_thresh_segment_batch(worker_type):
+    debug = worker_type == "thread"
+    path = retrieve_data(
+        data_path / "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+
+    # Get all the relevant information
+    with h5py.File(path) as h5:
+        image = h5["events/image"][:]
+        image_bg = h5["events/image_bg"][:]
+        mask = h5["events/mask"][:]
+        frame = h5["events/frame"][:]
+
+    # Concatenate the masks
+    frame_u, indices = np.unique(frame, return_index=True)
+    image_u = image[indices]
+    image_bg_u = image_bg[indices]
+    mask_u = np.zeros_like(image_u, dtype=bool)
+    for ii, fr in enumerate(frame):
+        idx = np.where(frame_u == fr)[0]
+        mask_u[idx] = np.logical_or(mask_u[idx], mask[ii])
+
+    image_u_c = np.array(image_u, dtype=int) - image_bg_u
+
+    sm = segm.segm_thresh.SegmentThresh(thresh=-6,
+                                        debug=debug,
+                                        kwargs_mask={"closing_disk": 3})
+
+    labels_seg = sm.segment_batch(image_u_c, start=0, stop=5)
+    assert labels_seg is sm.labels_array
+    assert np.all(np.array(labels_seg, dtype=bool) == sm.mask_array)
+    # tell workers to stop
+    sm.join_workers()
+
+    for ii in range(len(frame_u)):
+        mask_seg = np.array(labels_seg[ii], dtype=bool)
+        # Remove small objects, because this is not implemented in the
+        # segmenter class as it would be part of gating.
+        mask_seg = morphology.remove_small_objects(mask_seg, min_size=10)
+        assert np.all(mask_seg == mask_u[ii]), f"masks not matching at {ii}"
+
+
+@pytest.mark.parametrize("worker_type", ["thread", "process"])
+def test_segm_thresh_segment_batch_large(worker_type):
+    debug = worker_type == "thread"
+
+    # Create fake data
+    mask = np.zeros((121, 80, 200), dtype=bool)
+    mask[:, 10:71, 100:161] = morphology.disk(30).reshape(-1, 61, 61)
+    image = -10 * mask
+
+    sm = segm.segm_thresh.SegmentThresh(thresh=-6,
+                                        kwargs_mask={"closing_disk": 3},
+                                        debug=debug)
+
+    labels_seg_1 = np.copy(
+        sm.segment_batch(image, start=0, stop=101))
+
+    assert labels_seg_1.dtype == np.uint16  # uint8 is not enough
+    assert sm.mp_batch_index.value == 0
+    if worker_type == "thread":
+        assert len(sm._mp_workers) == 1
+        assert sm.mp_batch_worker.value == 1
+    else:
+        # This will fail if you have too many CPUs in your system
+        assert len(sm._mp_workers) == mp.cpu_count()
+        # Check whether all processes did their deeds
+        assert sm.mp_batch_worker.value == mp.cpu_count()
+
+    labels_seg_2 = np.copy(
+        sm.segment_batch(image, start=101, stop=121))
+
+    # tell workers to stop
+    sm.join_workers()
+
+    for ii in range(101):
+        mask_seg = np.array(labels_seg_1[ii], dtype=bool)
+        assert np.all(mask_seg == mask[ii]), f"masks not matching at {ii}"
+
+    for jj in range(101, 121):
+        mask_seg = np.array(labels_seg_2[jj - 101], dtype=bool)
+        assert np.all(mask_seg == mask[jj]), f"masks not matching at {jj}"

dcnum-0.11.1/tests/test_segm_thresh.py → dcnum-0.11.2/tests/test_segmenter.py

@@ -1,145 +1,36 @@
-import multiprocessing as mp
 import pathlib
 
 from dcnum import segm
-import h5py
 import numpy as np
-from skimage import morphology
-
-import pytest
-
-from helper_methods import retrieve_data
 
 data_path = pathlib.Path(__file__).parent / "data"
 
 
-def test_segm_thresh_basic():
-    """Basic thresholding segmenter
-
-    The segmenter is equivalent to the old dcevent legacy segmenter with
-    the options legacy:t=-6^bl=0^bi=0^d=1:cle=1^f=1^clo=3
-    (no blur, no binaryops, clear borders, fill holes, closing disk 3).
-    Since in the dcevent pipeline, the data are gated and small objects
-    are removed, we have to do this here manually before comparing mask
-    images.
-    """
-    path = retrieve_data(
-        data_path / "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
-
-    # Get all the relevant information
-    with h5py.File(path) as h5:
-        image = h5["events/image"][:]
-        image_bg = h5["events/image_bg"][:]
-        mask = h5["events/mask"][:]
-        frame = h5["events/frame"][:]
-
-    # Concatenate the masks
-    frame_u, indices = np.unique(frame, return_index=True)
-    image_u = image[indices]
-    image_bg_u = image_bg[indices]
-    mask_u = np.zeros_like(image_u, dtype=bool)
-    for ii, fr in enumerate(frame):
-        idx = np.where(frame_u == fr)[0]
-        mask_u[idx] = np.logical_or(mask_u[idx], mask[ii])
-
-    image_u_c = np.array(image_u, dtype=int) - image_bg_u
-
-    sm = segm.segm_thresh.SegmentThresh(thresh=-6,
-                                        kwargs_mask={"closing_disk": 3})
-    for ii in range(len(frame_u)):
-        labels_seg = sm.segment_frame(image_u_c[ii])
-        mask_seg = np.array(labels_seg, dtype=bool)
-        # Remove small objects, because this is not implemented in the
-        # segmenter class as it would be part of gating.
-        mask_seg = morphology.remove_small_objects(mask_seg, min_size=10)
-        assert np.all(mask_seg == mask_u[ii]), f"masks not matching at {ii}"
-
-
-@pytest.mark.parametrize("worker_type", ["thread", "process"])
-def test_segm_thresh_segment_batch(worker_type):
-    debug = worker_type == "thread"
-    path = retrieve_data(
-        data_path / "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
-
-    # Get all the relevant information
-    with h5py.File(path) as h5:
-        image = h5["events/image"][:]
-        image_bg = h5["events/image_bg"][:]
-        mask = h5["events/mask"][:]
-        frame = h5["events/frame"][:]
-
-    # Concatenate the masks
-    frame_u, indices = np.unique(frame, return_index=True)
-    image_u = image[indices]
-    image_bg_u = image_bg[indices]
-    mask_u = np.zeros_like(image_u, dtype=bool)
-    for ii, fr in enumerate(frame):
-        idx = np.where(frame_u == fr)[0]
-        mask_u[idx] = np.logical_or(mask_u[idx], mask[ii])
-
-    image_u_c = np.array(image_u, dtype=int) - image_bg_u
-
-    sm = segm.segm_thresh.SegmentThresh(thresh=-6,
-                                        debug=debug,
-                                        kwargs_mask={"closing_disk": 3})
-
-    labels_seg = sm.segment_batch(image_u_c, start=0, stop=5)
-    assert labels_seg is sm.labels_array
-    assert np.all(np.array(labels_seg, dtype=bool) == sm.mask_array)
-    # tell workers to stop
-    sm.join_workers()
-
-    for ii in range(len(frame_u)):
-        mask_seg = np.array(labels_seg[ii], dtype=bool)
-        # Remove small objects, because this is not implemented in the
-        # segmenter class as it would be part of gating.
-        mask_seg = morphology.remove_small_objects(mask_seg, min_size=10)
-        assert np.all(mask_seg == mask_u[ii]), f"masks not matching at {ii}"
-
-
-@pytest.mark.parametrize("worker_type", ["thread", "process"])
-def test_segm_thresh_segment_batch_large(worker_type):
-    debug = worker_type == "thread"
-
-    # Create fake data
-    mask = np.zeros((121, 80, 200), dtype=bool)
-    mask[:, 10:71, 100:161] = morphology.disk(30).reshape(-1, 61, 61)
-    image = -10 * mask
-
-    sm = segm.segm_thresh.SegmentThresh(thresh=-6,
-                                        kwargs_mask={"closing_disk": 3},
-                                        debug=debug)
-
-    labels_seg_1 = np.copy(
-        sm.segment_batch(image, start=0, stop=101))
-
-    assert labels_seg_1.dtype == np.uint16  # uint8 is not enough
-    assert sm.mp_batch_index.value == 0
-    if worker_type == "thread":
-        assert len(sm._mp_workers) == 1
-        assert sm.mp_batch_worker.value == 1
-    else:
-        # This will fail if you have too many CPUs in your system
-        assert len(sm._mp_workers) == mp.cpu_count()
-        # Check whether all processes did their deeds
-        assert sm.mp_batch_worker.value == mp.cpu_count()
-
-    labels_seg_2 = np.copy(
-        sm.segment_batch(image, start=101, stop=121))
-
-    # tell workers to stop
-    sm.join_workers()
-
-    for ii in range(101):
-        mask_seg = np.array(labels_seg_1[ii], dtype=bool)
-        assert np.all(mask_seg == mask[ii]), f"masks not matching at {ii}"
+class MockImageData:
+    mask = np.array([
+        [0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 1, 1, 1, 0, 0, 0],
+        [0, 0, 1, 0, 1, 0, 0, 0],  # filled, 1
+        [0, 0, 1, 1, 1, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 1, 1, 1, 1],
+        [0, 0, 0, 0, 0, 1, 1, 1],  # border, 2
+        [0, 0, 0, 0, 0, 1, 1, 1],
+        [0, 1, 1, 1, 0, 0, 0, 0],
+        [0, 0, 1, 1, 1, 0, 0, 0],  # other, 3
+        [0, 0, 1, 1, 1, 0, 0, 0],
+        [0, 0, 1, 1, 1, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0],
+        ], dtype=bool)
 
-    for jj in range(101, 121):
-        mask_seg = np.array(labels_seg_2[jj - 101], dtype=bool)
-        assert np.all(mask_seg == mask[jj]), f"masks not matching at {jj}"
+    def get_chunk(self, chunk_index):
+        image = np.array(-(10 + chunk_index) * self.mask, dtype=np.int16)
+        chunk = np.stack([image] * 100, dtype=np.int16)
+        return chunk
 
 
-def test_segm_thresh_labeled_mask():
+def test_segmenter_labeled_mask():
     mask = np.array([
         [0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 1, 1, 1, 0, 0, 0],
@@ -206,7 +97,7 @@ def test_segm_thresh_labeled_mask():
     assert np.sum(labels4 == 3) == 12
 
 
-def test_segm_thresh_labeled_mask_closing_disk():
+def test_segmenter_labeled_mask_closing_disk():
     mask = np.array([
         [0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0],
@@ -275,3 +166,60 @@ def test_segm_thresh_labeled_mask_closing_disk():
     assert np.sum(labels4 == 1) == 9
     assert np.sum(labels4 == 2) == 8
     assert np.sum(labels4 == 3) == 23
+
+
+def test_segmenter_labeled_mask_fill_holes_int32():
+    mask = np.array([
+        [0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 1, 1, 1, 0, 0, 0],
+        [0, 0, 1, 0, 1, 0, 0, 0],  # filled, 1
+        [0, 0, 1, 1, 1, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 1, 1, 1, 1],
+        [0, 0, 0, 0, 0, 1, 1, 1],  # border, 2
+        [0, 0, 0, 0, 0, 1, 1, 1],
+        [0, 1, 1, 1, 0, 0, 0, 0],
+        [0, 0, 1, 1, 1, 0, 0, 0],  # other, 3
+        [0, 0, 1, 1, 1, 0, 0, 0],
+        [0, 0, 1, 1, 1, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0],
+        ], dtype=bool)
+
+    sm1 = segm.segm_thresh.SegmentThresh(thresh=-6)
+    labels = np.array(sm1.segment_frame(-10 * mask), dtype=np.int64)
+    # sanity checks
+    assert labels.dtype == np.int64
+    assert labels.dtype != np.int32
+    labels_2 = sm1.process_mask(labels,
+                                clear_border=False,
+                                fill_holes=True,
+                                closing_disk=False)
+    assert np.allclose(labels, labels_2)
+    assert labels_2.dtype == np.int32
+
+
+def test_segmenter_segment_chunk():
+    with segm.segm_thresh.SegmentThresh(thresh=-12, debug=True) as sm:
+        image_data = MockImageData()
+        labels_1 = np.copy(sm.segment_chunk(image_data, 0))  # below threshold
+        assert sm.image_array.min() == -10
+        labels_2 = np.copy(sm.segment_chunk(image_data, 10))  # above threshold
+        assert sm.image_array.min() == -20
+        assert np.all(labels_1 == 0)
+        assert not np.all(labels_2 == 0)
+
+
+def test_cpu_segmenter_getsetstate():
+    sm1 = segm.segm_thresh.SegmentThresh(thresh=-12, debug=True)
+    with segm.segm_thresh.SegmentThresh(thresh=-12, debug=True) as sm2:
+        image_data = MockImageData()
+        # Do some processing so that we have workers
+        sm2.segment_chunk(image_data, 0)
+        # get the state
+        state = sm2.__getstate__()
+        # set the state
+        sm1.__setstate__(state)
+        # and here we test for the raw data that was transferred
+        assert not np.all(sm1.image_array == sm2.image_array)
+        assert np.all(sm1.mp_image_raw == sm2.mp_image_raw)

dcnum-0.11.1/tests/test_feat_haralick.py

@@ -1,33 +0,0 @@
-import pathlib
-
-import h5py
-import numpy as np
-
-from dcnum.feat import feat_texture
-
-from helper_methods import retrieve_data
-
-data_path = pathlib.Path(__file__).parent / "data"
-
-
-def test_basic_haralick():
-    # This original file was generated with dcevent for reference.
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
-    # Make data available
-    with h5py.File(path) as h5:
-        ret_arr = feat_texture.haralick_texture_features(
-            image=h5["events/image"][:],
-            image_bg=h5["events/image_bg"][:],
-            mask=h5["events/mask"][:],
-        )
-
-        assert np.allclose(ret_arr["tex_asm_avg"][1],
-                           0.001514295993357114,
-                           atol=0, rtol=1e-10)
-        for feat in feat_texture.haralick_names:
-            assert np.allclose(h5["events"][feat],
-                               ret_arr[feat])
-        # control test
-        assert not np.allclose(h5["events"]["tex_asm_avg"],
-                               ret_arr["tex_asm_ptp"])

dcnum-0.11.1/tests/test_feat_moments_based.py

@@ -1,52 +0,0 @@
-import pathlib
-
-import h5py
-import numpy as np
-
-from dcnum.feat import feat_moments
-
-from helper_methods import retrieve_data
-
-data_path = pathlib.Path(__file__).parent / "data"
-
-
-def test_moments_based_features():
-    # This original file was generated with dcevent for reference.
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
-    feats = [
-        "deform",
-        "size_x",
-        "size_y",
-        "pos_x",
-        "pos_y",
-        "area_msd",
-        "area_ratio",
-        "area_um",
-        "aspect",
-        "tilt",
-        "inert_ratio_cvx",
-        "inert_ratio_raw",
-        "inert_ratio_prnc",
-    ]
-
-    # Make data available
-    with h5py.File(path) as h5:
-        data = feat_moments.moments_based_features(
-            mask=h5["events/mask"][:],
-            pixel_size=0.2645
-        )
-        for feat in feats:
-            if feat.count("inert"):
-                rtol = 2e-5
-                atol = 1e-8
-            else:
-                rtol = 1e-5
-                atol = 1e-8
-            assert np.allclose(h5["events"][feat][:],
-                               data[feat],
-                               rtol=rtol,
-                               atol=atol), f"Feature {feat} mismatch!"
-        # control test
-        assert not np.allclose(h5["events"]["inert_ratio_cvx"][:],
-                               data["tilt"])