dcnum 0.23.1__tar.gz → 0.23.2__tar.gz

{dcnum-0.23.1 → dcnum-0.23.2}/CHANGELOG

@@ -1,3 +1,7 @@
+0.23.2
+ - enh: add DCNumPipelineJob.validate method
+ - enh: list Python libraries used in job log
+ - setup: change required pytorch version from 2.3 to 2.2 (hardware support)
 0.23.1
  - enh: support passing custom default arguments to get_class_method_info
  - tests: fix torch preprocessing tests

{dcnum-0.23.1 → dcnum-0.23.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.23.1
+Version: 0.23.2
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -26,7 +26,7 @@ Requires-Dist: opencv-python-headless
 Requires-Dist: scikit-image
 Requires-Dist: scipy>=1.8.0
 Provides-Extra: torch
-Requires-Dist: torch>=2.3; extra == "torch"
+Requires-Dist: torch>=2.2; extra == "torch"
 
 |dcnum|
 =======

{dcnum-0.23.1 → dcnum-0.23.2}/pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
 dynamic = ["version"]
 
 [project.optional-dependencies]
-torch = ["torch>=2.3"]
+torch = ["torch>=2.2"]
 
 [project.urls]
 source = "https://github.com/DC-Analysis/dcnum"

{dcnum-0.23.1 → dcnum-0.23.2}/src/dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.23.1'
-__version_tuple__ = version_tuple = (0, 23, 1)
+__version__ = version = '0.23.2'
+__version_tuple__ = version_tuple = (0, 23, 2)

{dcnum-0.23.1 → dcnum-0.23.2}/src/dcnum/logic/ctrl.py

@@ -1,6 +1,7 @@
 import collections
 import datetime
 import hashlib
+import importlib
 import json
 import logging
 from logging.handlers import QueueListener
@@ -430,6 +431,16 @@ class DCNumJobRunner(threading.Thread):
                                  "build": ", ".join(platform.python_build()),
                                  "implementation":
                                      platform.python_implementation(),
+                                 "libraries": get_library_versions_dict([
+                                     "cv2",
+                                     "h5py",
+                                     "mahotas",
+                                     "numba",
+                                     "numpy",
+                                     "scipy",
+                                     "skimage",
+                                     "torch",
+                                 ]),
                                  "version": platform.python_version(),
                                  },
                              "system": {
@@ -780,6 +791,19 @@ class DCNumJobRunner(threading.Thread):
         self.logger.info("Finished segmentation and feature extraction")
 
 
+def get_library_versions_dict(library_name_list):
+    version_dict = {}
+    for library_name in library_name_list:
+        try:
+            lib = importlib.import_module(library_name)
+        except BaseException:
+            version = None
+        else:
+            version = lib.__version__
+        version_dict[library_name] = version
+    return version_dict
+
+
 def join_thread_helper(thr, timeout, retries, logger, name):
     for _ in range(retries):
         thr.join(timeout=timeout)

{dcnum-0.23.1 → dcnum-0.23.2}/src/dcnum/logic/job.py

@@ -182,3 +182,25 @@ class DCNumPipelineJob:
         if len(ret) == 1:
             ret = ret[0]
         return ret
+
+    def validate(self):
+        """Make sure the pipeline will run given the job kwargs
+
+        Returns
+        -------
+        True:
+            for testing convenience
+
+        Raises
+        ------
+        dcnum.segm.SegmenterNotApplicableError:
+            the segmenter is incompatible with the input path
+        """
+        # Check segmenter applicability applicability
+        seg_cls = get_available_segmenters()[self.kwargs["segmenter_code"]]
+        with HDF5Data(self.kwargs["path_in"]) as hd:
+            seg_cls.validate_applicability(
+                segmenter_kwargs=self.kwargs["segmenter_kwargs"],
+                logs=hd.logs,
+                meta=hd.meta)
+        return True

{dcnum-0.23.1 → dcnum-0.23.2}/src/dcnum/segm/segm_torch/__init__.py

@@ -1,16 +1,20 @@
 import importlib
+import warnings
 
 try:
     torch = importlib.import_module("torch")
     req_maj = 2
-    req_min = 3
+    req_min = 2
     ver_tuple = torch.__version__.split(".")
     act_maj = int(ver_tuple[0])
     act_min = int(ver_tuple[1])
     if act_maj < req_maj or (act_maj == req_maj and act_min < req_min):
-        raise ValueError(f"Your PyTorch version {act_maj}.{act_min} is not "
-                         f"supported, please update to at least "
-                         f"{req_maj}.{req_min}")
+        warnings.warn(f"Your PyTorch version {act_maj}.{act_min} is "
+                      f"not supported, please update to at least "
+                      f"{req_maj}.{req_min} to use dcnum's PyTorch"
+                      f"segmenters")
+        raise ImportError(
+            f"Could not find PyTorch {req_maj}.{req_min}")
 except ImportError:
     pass
 else:

{dcnum-0.23.1 → dcnum-0.23.2}/src/dcnum.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.23.1
+Version: 0.23.2
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -26,7 +26,7 @@ Requires-Dist: opencv-python-headless
 Requires-Dist: scikit-image
 Requires-Dist: scipy>=1.8.0
 Provides-Extra: torch
-Requires-Dist: torch>=2.3; extra == "torch"
+Requires-Dist: torch>=2.2; extra == "torch"
 
 |dcnum|
 =======

{dcnum-0.23.1 → dcnum-0.23.2}/src/dcnum.egg-info/requires.txt

@@ -8,4 +8,4 @@ scikit-image
 scipy>=1.8.0
 
 [torch]
-torch>=2.3
+torch>=2.2

{dcnum-0.23.1 → dcnum-0.23.2}/tests/helper_methods.py

@@ -21,7 +21,7 @@ class MockImageData:
         [0, 0, 1, 1, 1, 0, 0, 0],
         [0, 0, 1, 1, 1, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0],
-        ], dtype=bool)
+    ], dtype=bool)
 
     def get_chunk(self, chunk_index):
         image = np.array(-(10 + chunk_index) * self.mask, dtype=np.int16)

{dcnum-0.23.1 → dcnum-0.23.2}/tests/test_feat_brightness.py

@@ -1,5 +1,3 @@
-import pathlib
-
 import h5py
 import numpy as np
 
@@ -7,13 +5,10 @@ from dcnum.feat import feat_brightness
 
 from helper_methods import retrieve_data
 
-data_path = pathlib.Path(__file__).parent / "data"
-
 
 def test_basic_brightness():
     # This original file was generated with dcevent for reference.
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     # Make data available
     with h5py.File(path) as h5:
         data = feat_brightness.brightness_features(
@@ -35,8 +30,7 @@ def test_basic_brightness():
 
 def test_basic_brightness_single_image():
     # This original file was generated with dcevent for reference.
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     # Make data available
     with h5py.File(path) as h5:
         data = feat_brightness.brightness_features(

{dcnum-0.23.1 → dcnum-0.23.2}/tests/test_feat_haralick.py

@@ -1,5 +1,3 @@
-import pathlib
-
 import h5py
 import numpy as np
 
@@ -7,13 +5,10 @@ from dcnum.feat import feat_texture
 
 from helper_methods import retrieve_data
 
-data_path = pathlib.Path(__file__).parent / "data"
-
 
 def test_basic_haralick():
     # This original file was generated with dcevent for reference.
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     # Make data available
     with h5py.File(path) as h5:
         ret_arr = feat_texture.haralick_texture_features(

{dcnum-0.23.1 → dcnum-0.23.2}/tests/test_feat_moments_based.py

@@ -1,5 +1,3 @@
-import pathlib
-
 import h5py
 import numpy as np
 import scipy.ndimage as ndi
@@ -8,8 +6,6 @@ from dcnum.feat import feat_contour
 
 from helper_methods import retrieve_data
 
-data_path = pathlib.Path(__file__).parent / "data"
-
 
 def test_inert_ratio_prnc():
     """Test tilt and equivalence of inert_ratio_raw and inert_ratio_prnc"""
@@ -151,8 +147,7 @@ def test_inert_ratio_prnc_simple_2():
 
 def test_moments_based_features():
     # This original file was generated with dcevent for reference.
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     feats = [
         "deform",
         "size_x",
@@ -201,9 +196,9 @@ def test_mask_0d():
         [0, 0, 0, 0, 0, 0],
     ], dtype=bool)[np.newaxis]
     data = feat_contour.moments_based_features(
-                mask=masks,
-                pixel_size=0.2645
-            )
+        mask=masks,
+        pixel_size=0.2645
+    )
     assert data["deform"].shape == (1,)
     assert np.isnan(data["deform"][0])
     assert np.isnan(data["area_um"][0])
@@ -219,9 +214,9 @@ def test_mask_1d():
         [0, 0, 0, 0, 0, 0],
     ], dtype=bool)[np.newaxis]
     data = feat_contour.moments_based_features(
-                mask=masks,
-                pixel_size=0.2645
-            )
+        mask=masks,
+        pixel_size=0.2645
+    )
     assert data["deform"].shape == (1,)
     assert np.isnan(data["deform"][0])
     assert np.isnan(data["area_um"][0])
@@ -237,9 +232,9 @@ def test_mask_1d_large():
         [0, 0, 1, 0, 0, 0],
     ], dtype=bool)[np.newaxis]
     data = feat_contour.moments_based_features(
-                mask=masks,
-                pixel_size=0.2645
-            )
+        mask=masks,
+        pixel_size=0.2645
+    )
     assert data["deform"].shape == (1,)
     assert np.isnan(data["deform"][0])
     assert np.isnan(data["area_um"][0])
@@ -257,9 +252,9 @@ def test_mask_1d_large_no_border():
         [0, 0, 0, 0, 0, 0],
     ], dtype=bool)[np.newaxis]
     data = feat_contour.moments_based_features(
-                mask=masks,
-                pixel_size=0.2645
-            )
+        mask=masks,
+        pixel_size=0.2645
+    )
     assert data["deform"].shape == (1,)
     assert np.isnan(data["deform"][0])
     assert np.isnan(data["area_um"][0])
@@ -275,9 +270,9 @@ def test_mask_2d():
         [0, 0, 0, 0, 0, 0],
     ], dtype=bool)[np.newaxis]
     data = feat_contour.moments_based_features(
-                mask=masks,
-                pixel_size=0.2645
-            )
+        mask=masks,
+        pixel_size=0.2645
+    )
     assert data["deform"].shape == (1,)
     # This is the deformation of a square (compared to circle)
     assert np.allclose(data["deform"][0], 0.11377307454724206)
@@ -305,8 +300,8 @@ def test_mask_mixed():
     mixed_masks = np.append(mask_valid[None, ...],
                             mask_invalid[None, ...], axis=0)
     data = feat_contour.moments_based_features(
-                mask=mixed_masks,
-                pixel_size=0.2645)
+        mask=mixed_masks,
+        pixel_size=0.2645)
     assert data["deform"].shape == (2,)
     assert np.all(data["valid"][:] == np.array([True, False]))
     assert not np.isnan(data["deform"][0])

{dcnum-0.23.1 → dcnum-0.23.2}/tests/test_feat_moments_based_extended.py

@@ -1,5 +1,3 @@
-import pathlib
-
 import h5py
 import numpy as np
 
@@ -7,21 +5,18 @@ from dcnum.feat import feat_contour
 
 from helper_methods import retrieve_data
 
-data_path = pathlib.Path(__file__).parent / "data"
-
 
 def test_moments_based_features():
     # This file has new cell features belonging to
     # fmt-hdf5_cytoshot_full-features_2023.zip
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_extended-moments-features.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_extended-moments-features.zip")
 
     feats = [
-            "area_um_raw",
-            "deform_raw",
-            "eccentr_prnc",
-            "per_ratio",
-            "per_um_raw",
+        "area_um_raw",
+        "deform_raw",
+        "eccentr_prnc",
+        "per_ratio",
+        "per_um_raw",
     ]
 
     # Make data available
@@ -49,9 +44,9 @@ def test_mask_2d():
         [0, 0, 0, 0, 0, 0],
     ], dtype=bool)[np.newaxis]
     data = feat_contour.moments_based_features(
-                mask=masks,
-                pixel_size=0.2645
-            )
+        mask=masks,
+        pixel_size=0.2645
+    )
     assert data["deform_raw"].shape == (1,)
     # This is the deformation of a square (compared to circle)
     assert np.allclose(data["deform_raw"][0], 0.11377307454724206)

{dcnum-0.23.1 → dcnum-0.23.2}/tests/test_feat_volume.py

@@ -1,5 +1,4 @@
 import itertools
-import pathlib
 
 import h5py
 import numpy as np
@@ -10,8 +9,6 @@ from dcnum.feat.feat_contour import moments_based_features
 
 from helper_methods import retrieve_data
 
-data_path = pathlib.Path(__file__).parent / "data"
-
 
 def area_of_polygon(x, y):
     return 0.5 * np.abs(np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1)))
@@ -105,8 +102,7 @@ def test_volume_from_file():
         1.09564687e+02, 2.39239370e-01, 1.71917437e-01, 8.98323862e+01,
         4.13412223e+00, 2.91659170e+02, 2.00198054e+02, 1.97545320e+00,
         9.15408837e+01, 1.60965362e-01, 3.48553309e-01, 2.04561447e+02])
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
 
     with h5py.File(path) as h5:
         pixel_size = h5.attrs["imaging:pixel size"]

{dcnum-0.23.1 → dcnum-0.23.2}/tests/test_logic_job.py

@@ -1,8 +1,12 @@
 import multiprocessing as mp
 
 from dcnum import logic
+from dcnum.segm.segm_torch import segm_torch_base  # noqa: E402
+import h5py
 
-from helper_methods import retrieve_data
+import pytest
+
+from helper_methods import retrieve_data, retrieve_model
 
 
 def test_basic_job():
@@ -36,3 +40,35 @@ def test_segmenter_mask():
                                  )
     _, pdict = job.get_ppid(ret_dict=True)
     assert pdict["seg_id"] == "thresh:t=-6:cle=1^f=1^clo=3"
+
+
+def test_validate_invalid_model():
+    model_file = retrieve_model(
+        "segm-torch-model_unet-dcnum-test_g1_910c2.zip")
+
+    # Create a test dataset with metadata that will make the model invalid
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2024.zip")
+
+    with h5py.File(path, "a") as h5:
+        h5.attrs["setup:chip region"] = "reservoir"
+
+    job = logic.DCNumPipelineJob(path_in=path,
+                                 segmenter_code="torchmpo",
+                                 segmenter_kwargs={
+                                     "model_file": model_file},
+                                 )
+
+    with pytest.raises(
+            segm_torch_base.SegmenterNotApplicableError,
+            match="only experiments in channel region supported"):
+        job.validate()
+
+
+def test_validate_ok():
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
+    job = logic.DCNumPipelineJob(path_in=path,
+                                 segmenter_code="thresh",
+                                 segmenter_kwargs={
+                                     "kwargs_mask": {"closing_disk": 3}},
+                                 )
+    assert job.validate()

{dcnum-0.23.1 → dcnum-0.23.2}/tests/test_logic_pipeline.py

@@ -135,7 +135,7 @@ def test_basin_strategy_tap():
 
     with h5py.File(path_out) as h5:
         assert h5.attrs["pipeline:dcnum background"] \
-               == "sparsemed:k=150^s=1^t=0^f=0.8^o=1"
+            == "sparsemed:k=150^s=1^t=0^f=0.8^o=1"
         assert "image_bg" in h5["events"]
         assert "bg_off" in h5["events"]
         assert "deform" in h5["events"]
@@ -174,7 +174,7 @@ def test_basin_relative_path():
     # Everything should just work, because we have relative paths in the basin.
     with h5py.File(path_out_new) as h5:
         assert h5.attrs["pipeline:dcnum background"] \
-               == "sparsemed:k=150^s=1^t=0^f=0.8^o=1"
+            == "sparsemed:k=150^s=1^t=0^f=0.8^o=1"
         assert "image_bg" in h5["events"]
         assert "bg_off" in h5["events"]
         assert "deform" in h5["events"]
@@ -207,7 +207,7 @@ def test_chained_pipeline():
 
     with h5py.File(path2) as h5:
         assert h5.attrs["pipeline:dcnum background"] \
-               == "sparsemed:k=150^s=1^t=0^f=0.8^o=1"
+            == "sparsemed:k=150^s=1^t=0^f=0.8^o=1"
         assert "image" in h5["events"]
         assert "image_bg" in h5["events"]
         for feat in h5["events"]:
@@ -228,7 +228,7 @@ def test_chained_pipeline():
         assert "image_bg" in h5["events"]
         assert len(h5["events/deform"]) == 285
         assert h5.attrs["pipeline:dcnum background"] \
-               == "sparsemed:k=250^s=1^t=0^f=0.8^o=1"
+            == "sparsemed:k=250^s=1^t=0^f=0.8^o=1"
         for feat in h5["events"]:
             assert len(h5["events"][feat]) == 285
 
@@ -745,8 +745,8 @@ def test_simple_pipeline(debug):
         assert h5.attrs["experiment:sample"] == "data"
         assert h5.attrs["experiment:date"] == "2022-04-21"
         assert h5.attrs["experiment:run identifier"] == \
-               (f"d5a40aed-0b6c-0412-e87c-59789fdd28d0_"
-                f"dcn-{pp_hash[:7]}")
+            (f"d5a40aed-0b6c-0412-e87c-59789fdd28d0_"
+             f"dcn-{pp_hash[:7]}")
 
 
 @pytest.mark.parametrize("debug", [True, False])
@@ -814,8 +814,8 @@ def test_simple_pipeline_no_offset_correction(debug):
         assert h5.attrs["experiment:sample"] == "data"
         assert h5.attrs["experiment:date"] == "2022-04-21"
         assert h5.attrs["experiment:run identifier"] == \
-               (f"d5a40aed-0b6c-0412-e87c-59789fdd28d0_"
-                f"dcn-{pp_hash[:7]}")
+            (f"d5a40aed-0b6c-0412-e87c-59789fdd28d0_"
+             f"dcn-{pp_hash[:7]}")
 
 
 def test_simple_pipeline_in_thread():

{dcnum-0.23.1 → dcnum-0.23.2}/tests/test_read_concat_hdf5.py

@@ -1,5 +1,3 @@
-import pathlib
-
 from dcnum import read
 import h5py
 import numpy as np
@@ -7,13 +5,10 @@ import pytest
 
 from helper_methods import retrieve_data
 
-data_path = pathlib.Path(__file__).parent / "data"
-
 
 @pytest.mark.parametrize("path_out", [None, True])
 def test_concat_basic(path_out):
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     # create simple concatenated dataset, repeating a file
     data = read.concatenated_hdf5_data([path, path, path],
                                        path_out=path_out)
@@ -22,8 +17,7 @@ def test_concat_basic(path_out):
 
 @pytest.mark.parametrize("path_out", [None, True])
 def test_concat_basic_frame(path_out):
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     # create simple concatenated dataset, repeating a file
     data = read.concatenated_hdf5_data([path, path, path],
                                        path_out=path_out)
@@ -43,8 +37,7 @@ def test_concat_basic_frame(path_out):
 
 
 def test_concat_basic_to_file(tmp_path):
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     # create simple concatenated dataset, repeating a file
     path_out = tmp_path / "test.rtdc"
     assert not path_out.exists()
@@ -56,8 +49,7 @@ def test_concat_basic_to_file(tmp_path):
 
 def test_concat_ignore_contour(tmp_path):
     # get file wtih contour information
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     cdata = np.array([[2, 2], [2, 3], [2, 5], [4, 5], [4, 2]])
     with h5py.File(path, mode="a") as h5:
         contour = h5["events"].create_group("contour")
@@ -76,10 +68,8 @@ def test_concat_ignore_contour(tmp_path):
 
 
 def test_concat_invalid_input_feature_number():
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
-    path2 = retrieve_data(data_path /
-                          "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
+    path2 = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
 
     # Add a new feature to path2
     with h5py.File(path2, mode="a") as h5:
@@ -91,8 +81,7 @@ def test_concat_invalid_input_feature_number():
 
 
 def test_concat_invalid_input_path():
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     invalid_output = 42
     with pytest.raises(ValueError, match="Invalid type"):
         read.concatenated_hdf5_data([path, path, path],
@@ -100,17 +89,14 @@ def test_concat_invalid_input_path():
 
 
 def test_concat_invalid_input_path_number():
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     with pytest.raises(ValueError, match="Please specify at least two"):
         read.concatenated_hdf5_data([path])
 
 
 def test_concat_specify_input_feature_number():
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
-    path2 = retrieve_data(data_path /
-                          "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
+    path2 = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
 
     # Add a new feature to path2
     with h5py.File(path2, mode="a") as h5: