dcnum 0.23.0__tar.gz → 0.23.2__tar.gz

{dcnum-0.23.0 → dcnum-0.23.2}/.github/workflows/check.yml

@@ -31,7 +31,7 @@ jobs:
       run: |
         # https://github.com/luispedro/mahotas/issues/144
         pip install mahotas==1.4.13
-        pip install -e .
+        pip install .[torch]
     - name: List installed packages
       run: |
         pip freeze

{dcnum-0.23.0 → dcnum-0.23.2}/CHANGELOG

@@ -1,3 +1,10 @@
+0.23.2
+ - enh: add DCNumPipelineJob.validate method
+ - enh: list Python libraries used in job log
+ - setup: change required pytorch version from 2.3 to 2.2 (hardware support)
+0.23.1
+ - enh: support passing custom default arguments to get_class_method_info
+ - tests: fix torch preprocessing tests
 0.23.0
  - feat: implement segmentation using PyTorch models
  - fix: always compute image_bg if it is not in the input file

{dcnum-0.23.0 → dcnum-0.23.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.23.0
+Version: 0.23.2
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -26,7 +26,7 @@ Requires-Dist: opencv-python-headless
 Requires-Dist: scikit-image
 Requires-Dist: scipy>=1.8.0
 Provides-Extra: torch
-Requires-Dist: torch>=2.3; extra == "torch"
+Requires-Dist: torch>=2.2; extra == "torch"
 
 |dcnum|
 =======

{dcnum-0.23.0 → dcnum-0.23.2}/pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
 dynamic = ["version"]
 
 [project.optional-dependencies]
-torch = ["torch>=2.3"]
+torch = ["torch>=2.2"]
 
 [project.urls]
 source = "https://github.com/DC-Analysis/dcnum"

{dcnum-0.23.0 → dcnum-0.23.2}/src/dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.23.0'
-__version_tuple__ = version_tuple = (0, 23, 0)
+__version__ = version = '0.23.2'
+__version_tuple__ = version_tuple = (0, 23, 2)

{dcnum-0.23.0 → dcnum-0.23.2}/src/dcnum/logic/ctrl.py

@@ -1,6 +1,7 @@
 import collections
 import datetime
 import hashlib
+import importlib
 import json
 import logging
 from logging.handlers import QueueListener
@@ -430,6 +431,16 @@ class DCNumJobRunner(threading.Thread):
                                  "build": ", ".join(platform.python_build()),
                                  "implementation":
                                      platform.python_implementation(),
+                                 "libraries": get_library_versions_dict([
+                                     "cv2",
+                                     "h5py",
+                                     "mahotas",
+                                     "numba",
+                                     "numpy",
+                                     "scipy",
+                                     "skimage",
+                                     "torch",
+                                 ]),
                                  "version": platform.python_version(),
                                  },
                              "system": {
@@ -780,6 +791,19 @@ class DCNumJobRunner(threading.Thread):
         self.logger.info("Finished segmentation and feature extraction")
 
 
+def get_library_versions_dict(library_name_list):
+    version_dict = {}
+    for library_name in library_name_list:
+        try:
+            lib = importlib.import_module(library_name)
+        except BaseException:
+            version = None
+        else:
+            version = lib.__version__
+        version_dict[library_name] = version
+    return version_dict
+
+
 def join_thread_helper(thr, timeout, retries, logger, name):
     for _ in range(retries):
         thr.join(timeout=timeout)

{dcnum-0.23.0 → dcnum-0.23.2}/src/dcnum/logic/job.py

@@ -182,3 +182,25 @@ class DCNumPipelineJob:
         if len(ret) == 1:
             ret = ret[0]
         return ret
+
+    def validate(self):
+        """Make sure the pipeline will run given the job kwargs
+
+        Returns
+        -------
+        True:
+            for testing convenience
+
+        Raises
+        ------
+        dcnum.segm.SegmenterNotApplicableError:
+            the segmenter is incompatible with the input path
+        """
+        # Check segmenter applicability applicability
+        seg_cls = get_available_segmenters()[self.kwargs["segmenter_code"]]
+        with HDF5Data(self.kwargs["path_in"]) as hd:
+            seg_cls.validate_applicability(
+                segmenter_kwargs=self.kwargs["segmenter_kwargs"],
+                logs=hd.logs,
+                meta=hd.meta)
+        return True

{dcnum-0.23.0 → dcnum-0.23.2}/src/dcnum/meta/ppid.py

@@ -59,7 +59,9 @@ def convert_to_dtype(value, dtype):
 
 
 def get_class_method_info(class_obj: ClassWithPPIDCapabilities,
-                          static_kw_methods: List = None):
+                          static_kw_methods: List = None,
+                          static_kw_defaults: Dict = None,
+                          ):
     """Return dictionary of class info with static keyword methods docs
 
     Parameters
@@ -69,7 +71,16 @@ def get_class_method_info(class_obj: ClassWithPPIDCapabilities,
     static_kw_methods: list of callable
         The methods to inspect; all kwargs-only keyword arguments
         are extracted.
+    static_kw_defaults: dict
+        If a key in this dictionary matches an item in `static_kw_methods`,
+        then these are the default values returned in the "defaults"
+        dictionary. This is used in cases where a base class does
+        implement some annotations, but the subclass does not actually
+        use them, because e.g. they are taken from a property such as is
+        the case for the mask postprocessing of segmenter classes.
     """
+    if static_kw_defaults is None:
+        static_kw_defaults = {}
     doc = class_obj.__doc__ or class_obj.__init__.__doc__
     info = {
         "code": class_obj.get_ppid_code(),
@@ -82,7 +93,10 @@ def get_class_method_info(class_obj: ClassWithPPIDCapabilities,
         for mm in static_kw_methods:
             meth = getattr(class_obj, mm)
             spec = inspect.getfullargspec(meth)
-            defau[mm] = spec.kwonlydefaults or {}
+            if mm_defaults := static_kw_defaults.get(mm):
+                defau[mm] = mm_defaults
+            else:
+                defau[mm] = spec.kwonlydefaults or {}
             annot[mm] = spec.annotations
         info["defaults"] = defau
         info["annotations"] = annot

{dcnum-0.23.0 → dcnum-0.23.2}/src/dcnum/segm/segm_torch/__init__.py

@@ -1,16 +1,20 @@
 import importlib
+import warnings
 
 try:
     torch = importlib.import_module("torch")
     req_maj = 2
-    req_min = 3
+    req_min = 2
     ver_tuple = torch.__version__.split(".")
     act_maj = int(ver_tuple[0])
     act_min = int(ver_tuple[1])
     if act_maj < req_maj or (act_maj == req_maj and act_min < req_min):
-        raise ValueError(f"Your PyTorch version {act_maj}.{act_min} is not "
-                         f"supported, please update to at least "
-                         f"{req_maj}.{req_min}")
+        warnings.warn(f"Your PyTorch version {act_maj}.{act_min} is "
+                      f"not supported, please update to at least "
+                      f"{req_maj}.{req_min} to use dcnum's PyTorch"
+                      f"segmenters")
+        raise ImportError(
+            f"Could not find PyTorch {req_maj}.{req_min}")
 except ImportError:
     pass
 else:

{dcnum-0.23.0 → dcnum-0.23.2}/src/dcnum/segm/segm_torch/torch_preproc.py

@@ -4,8 +4,8 @@ import numpy as np
 
 
 def preprocess_images(images: np.ndarray,
-                      norm_mean: float,
-                      norm_std: float,
+                      norm_mean: float | None,
+                      norm_std: float | None,
                       image_shape: Tuple[int, int] = None,
                       ):
     """Transform image data to something torch models expect
@@ -24,10 +24,11 @@ def preprocess_images(images: np.ndarray,
         2D image, it will be reshaped to a 3D image with a batch_size of 1.
     norm_mean:
         Mean value used for standard score data normalization, i.e.
-        `normalized = `(images / 255 - norm_mean) / norm_std`
+        `normalized = `(images / 255 - norm_mean) / norm_std`; Set
+        to None to disable normalization.
     norm_std:
-        Standard deviation used for standard score data normalization
-        (see above)
+        Standard deviation used for standard score data normalization;
+        Set to None to disable normalization (see above).
     image_shape
         Image shape for which the model was created (height, width).
         If the image shape does not match the input image shape, then
@@ -102,8 +103,12 @@ def preprocess_images(images: np.ndarray,
         # Replace img_norm
         img_proc = img_pad
 
-    # normalize images
-    img_norm = (img_proc.astype(np.float32) / 255 - norm_mean) / norm_std
+    if norm_mean is None or norm_std is None:
+        # convert to float32
+        img_norm = img_proc.astype(np.float32)
+    else:
+        # normalize images
+        img_norm = (img_proc.astype(np.float32) / 255 - norm_mean) / norm_std
 
     # Add a "channels" axis for the ML models.
     return img_norm[:, np.newaxis, :, :]

{dcnum-0.23.0 → dcnum-0.23.2}/src/dcnum.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.23.0
+Version: 0.23.2
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -26,7 +26,7 @@ Requires-Dist: opencv-python-headless
 Requires-Dist: scikit-image
 Requires-Dist: scipy>=1.8.0
 Provides-Extra: torch
-Requires-Dist: torch>=2.3; extra == "torch"
+Requires-Dist: torch>=2.2; extra == "torch"
 
 |dcnum|
 =======

{dcnum-0.23.0 → dcnum-0.23.2}/src/dcnum.egg-info/requires.txt

@@ -8,4 +8,4 @@ scikit-image
 scipy>=1.8.0
 
 [torch]
-torch>=2.3
+torch>=2.2

{dcnum-0.23.0 → dcnum-0.23.2}/tests/helper_methods.py

@@ -21,7 +21,7 @@ class MockImageData:
         [0, 0, 1, 1, 1, 0, 0, 0],
         [0, 0, 1, 1, 1, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0],
-        ], dtype=bool)
+    ], dtype=bool)
 
     def get_chunk(self, chunk_index):
         image = np.array(-(10 + chunk_index) * self.mask, dtype=np.int16)

{dcnum-0.23.0 → dcnum-0.23.2}/tests/test_feat_brightness.py

@@ -1,5 +1,3 @@
-import pathlib
-
 import h5py
 import numpy as np
 
@@ -7,13 +5,10 @@ from dcnum.feat import feat_brightness
 
 from helper_methods import retrieve_data
 
-data_path = pathlib.Path(__file__).parent / "data"
-
 
 def test_basic_brightness():
     # This original file was generated with dcevent for reference.
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     # Make data available
     with h5py.File(path) as h5:
         data = feat_brightness.brightness_features(
@@ -35,8 +30,7 @@ def test_basic_brightness():
 
 def test_basic_brightness_single_image():
     # This original file was generated with dcevent for reference.
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     # Make data available
     with h5py.File(path) as h5:
         data = feat_brightness.brightness_features(

{dcnum-0.23.0 → dcnum-0.23.2}/tests/test_feat_haralick.py

@@ -1,5 +1,3 @@
-import pathlib
-
 import h5py
 import numpy as np
 
@@ -7,13 +5,10 @@ from dcnum.feat import feat_texture
 
 from helper_methods import retrieve_data
 
-data_path = pathlib.Path(__file__).parent / "data"
-
 
 def test_basic_haralick():
     # This original file was generated with dcevent for reference.
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     # Make data available
     with h5py.File(path) as h5:
         ret_arr = feat_texture.haralick_texture_features(

{dcnum-0.23.0 → dcnum-0.23.2}/tests/test_feat_moments_based.py

@@ -1,5 +1,3 @@
-import pathlib
-
 import h5py
 import numpy as np
 import scipy.ndimage as ndi
@@ -8,8 +6,6 @@ from dcnum.feat import feat_contour
 
 from helper_methods import retrieve_data
 
-data_path = pathlib.Path(__file__).parent / "data"
-
 
 def test_inert_ratio_prnc():
     """Test tilt and equivalence of inert_ratio_raw and inert_ratio_prnc"""
@@ -151,8 +147,7 @@ def test_inert_ratio_prnc_simple_2():
 
 def test_moments_based_features():
     # This original file was generated with dcevent for reference.
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
     feats = [
         "deform",
         "size_x",
@@ -201,9 +196,9 @@ def test_mask_0d():
         [0, 0, 0, 0, 0, 0],
     ], dtype=bool)[np.newaxis]
     data = feat_contour.moments_based_features(
-                mask=masks,
-                pixel_size=0.2645
-            )
+        mask=masks,
+        pixel_size=0.2645
+    )
     assert data["deform"].shape == (1,)
     assert np.isnan(data["deform"][0])
     assert np.isnan(data["area_um"][0])
@@ -219,9 +214,9 @@ def test_mask_1d():
         [0, 0, 0, 0, 0, 0],
     ], dtype=bool)[np.newaxis]
     data = feat_contour.moments_based_features(
-                mask=masks,
-                pixel_size=0.2645
-            )
+        mask=masks,
+        pixel_size=0.2645
+    )
     assert data["deform"].shape == (1,)
     assert np.isnan(data["deform"][0])
     assert np.isnan(data["area_um"][0])
@@ -237,9 +232,9 @@ def test_mask_1d_large():
         [0, 0, 1, 0, 0, 0],
     ], dtype=bool)[np.newaxis]
     data = feat_contour.moments_based_features(
-                mask=masks,
-                pixel_size=0.2645
-            )
+        mask=masks,
+        pixel_size=0.2645
+    )
     assert data["deform"].shape == (1,)
     assert np.isnan(data["deform"][0])
     assert np.isnan(data["area_um"][0])
@@ -257,9 +252,9 @@ def test_mask_1d_large_no_border():
         [0, 0, 0, 0, 0, 0],
     ], dtype=bool)[np.newaxis]
     data = feat_contour.moments_based_features(
-                mask=masks,
-                pixel_size=0.2645
-            )
+        mask=masks,
+        pixel_size=0.2645
+    )
     assert data["deform"].shape == (1,)
     assert np.isnan(data["deform"][0])
     assert np.isnan(data["area_um"][0])
@@ -275,9 +270,9 @@ def test_mask_2d():
         [0, 0, 0, 0, 0, 0],
     ], dtype=bool)[np.newaxis]
     data = feat_contour.moments_based_features(
-                mask=masks,
-                pixel_size=0.2645
-            )
+        mask=masks,
+        pixel_size=0.2645
+    )
     assert data["deform"].shape == (1,)
     # This is the deformation of a square (compared to circle)
     assert np.allclose(data["deform"][0], 0.11377307454724206)
@@ -305,8 +300,8 @@ def test_mask_mixed():
     mixed_masks = np.append(mask_valid[None, ...],
                             mask_invalid[None, ...], axis=0)
     data = feat_contour.moments_based_features(
-                mask=mixed_masks,
-                pixel_size=0.2645)
+        mask=mixed_masks,
+        pixel_size=0.2645)
     assert data["deform"].shape == (2,)
     assert np.all(data["valid"][:] == np.array([True, False]))
     assert not np.isnan(data["deform"][0])

{dcnum-0.23.0 → dcnum-0.23.2}/tests/test_feat_moments_based_extended.py

@@ -1,5 +1,3 @@
-import pathlib
-
 import h5py
 import numpy as np
 
@@ -7,21 +5,18 @@ from dcnum.feat import feat_contour
 
 from helper_methods import retrieve_data
 
-data_path = pathlib.Path(__file__).parent / "data"
-
 
 def test_moments_based_features():
     # This file has new cell features belonging to
     # fmt-hdf5_cytoshot_full-features_2023.zip
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_extended-moments-features.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_extended-moments-features.zip")
 
     feats = [
-            "area_um_raw",
-            "deform_raw",
-            "eccentr_prnc",
-            "per_ratio",
-            "per_um_raw",
+        "area_um_raw",
+        "deform_raw",
+        "eccentr_prnc",
+        "per_ratio",
+        "per_um_raw",
     ]
 
     # Make data available
@@ -49,9 +44,9 @@ def test_mask_2d():
         [0, 0, 0, 0, 0, 0],
     ], dtype=bool)[np.newaxis]
     data = feat_contour.moments_based_features(
-                mask=masks,
-                pixel_size=0.2645
-            )
+        mask=masks,
+        pixel_size=0.2645
+    )
     assert data["deform_raw"].shape == (1,)
     # This is the deformation of a square (compared to circle)
     assert np.allclose(data["deform_raw"][0], 0.11377307454724206)

{dcnum-0.23.0 → dcnum-0.23.2}/tests/test_feat_volume.py

@@ -1,5 +1,4 @@
 import itertools
-import pathlib
 
 import h5py
 import numpy as np
@@ -10,8 +9,6 @@ from dcnum.feat.feat_contour import moments_based_features
 
 from helper_methods import retrieve_data
 
-data_path = pathlib.Path(__file__).parent / "data"
-
 
 def area_of_polygon(x, y):
     return 0.5 * np.abs(np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1)))
@@ -105,8 +102,7 @@ def test_volume_from_file():
         1.09564687e+02, 2.39239370e-01, 1.71917437e-01, 8.98323862e+01,
         4.13412223e+00, 2.91659170e+02, 2.00198054e+02, 1.97545320e+00,
         9.15408837e+01, 1.60965362e-01, 3.48553309e-01, 2.04561447e+02])
-    path = retrieve_data(data_path /
-                         "fmt-hdf5_cytoshot_full-features_2023.zip")
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
 
     with h5py.File(path) as h5:
         pixel_size = h5.attrs["imaging:pixel size"]

{dcnum-0.23.0 → dcnum-0.23.2}/tests/test_logic_job.py

@@ -1,8 +1,12 @@
 import multiprocessing as mp
 
 from dcnum import logic
+from dcnum.segm.segm_torch import segm_torch_base  # noqa: E402
+import h5py
 
-from helper_methods import retrieve_data
+import pytest
+
+from helper_methods import retrieve_data, retrieve_model
 
 
 def test_basic_job():
@@ -36,3 +40,35 @@ def test_segmenter_mask():
                                  )
     _, pdict = job.get_ppid(ret_dict=True)
     assert pdict["seg_id"] == "thresh:t=-6:cle=1^f=1^clo=3"
+
+
+def test_validate_invalid_model():
+    model_file = retrieve_model(
+        "segm-torch-model_unet-dcnum-test_g1_910c2.zip")
+
+    # Create a test dataset with metadata that will make the model invalid
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2024.zip")
+
+    with h5py.File(path, "a") as h5:
+        h5.attrs["setup:chip region"] = "reservoir"
+
+    job = logic.DCNumPipelineJob(path_in=path,
+                                 segmenter_code="torchmpo",
+                                 segmenter_kwargs={
+                                     "model_file": model_file},
+                                 )
+
+    with pytest.raises(
+            segm_torch_base.SegmenterNotApplicableError,
+            match="only experiments in channel region supported"):
+        job.validate()
+
+
+def test_validate_ok():
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
+    job = logic.DCNumPipelineJob(path_in=path,
+                                 segmenter_code="thresh",
+                                 segmenter_kwargs={
+                                     "kwargs_mask": {"closing_disk": 3}},
+                                 )
+    assert job.validate()

{dcnum-0.23.0 → dcnum-0.23.2}/tests/test_logic_pipeline.py

@@ -135,7 +135,7 @@ def test_basin_strategy_tap():
 
     with h5py.File(path_out) as h5:
         assert h5.attrs["pipeline:dcnum background"] \
-               == "sparsemed:k=150^s=1^t=0^f=0.8^o=1"
+            == "sparsemed:k=150^s=1^t=0^f=0.8^o=1"
         assert "image_bg" in h5["events"]
         assert "bg_off" in h5["events"]
         assert "deform" in h5["events"]
@@ -174,7 +174,7 @@ def test_basin_relative_path():
     # Everything should just work, because we have relative paths in the basin.
     with h5py.File(path_out_new) as h5:
         assert h5.attrs["pipeline:dcnum background"] \
-               == "sparsemed:k=150^s=1^t=0^f=0.8^o=1"
+            == "sparsemed:k=150^s=1^t=0^f=0.8^o=1"
         assert "image_bg" in h5["events"]
         assert "bg_off" in h5["events"]
         assert "deform" in h5["events"]
@@ -207,7 +207,7 @@ def test_chained_pipeline():
 
     with h5py.File(path2) as h5:
         assert h5.attrs["pipeline:dcnum background"] \
-               == "sparsemed:k=150^s=1^t=0^f=0.8^o=1"
+            == "sparsemed:k=150^s=1^t=0^f=0.8^o=1"
         assert "image" in h5["events"]
         assert "image_bg" in h5["events"]
         for feat in h5["events"]:
@@ -228,7 +228,7 @@ def test_chained_pipeline():
         assert "image_bg" in h5["events"]
         assert len(h5["events/deform"]) == 285
         assert h5.attrs["pipeline:dcnum background"] \
-               == "sparsemed:k=250^s=1^t=0^f=0.8^o=1"
+            == "sparsemed:k=250^s=1^t=0^f=0.8^o=1"
         for feat in h5["events"]:
             assert len(h5["events"][feat]) == 285
 
@@ -745,8 +745,8 @@ def test_simple_pipeline(debug):
         assert h5.attrs["experiment:sample"] == "data"
         assert h5.attrs["experiment:date"] == "2022-04-21"
         assert h5.attrs["experiment:run identifier"] == \
-               (f"d5a40aed-0b6c-0412-e87c-59789fdd28d0_"
-                f"dcn-{pp_hash[:7]}")
+            (f"d5a40aed-0b6c-0412-e87c-59789fdd28d0_"
+             f"dcn-{pp_hash[:7]}")
 
 
 @pytest.mark.parametrize("debug", [True, False])
@@ -814,8 +814,8 @@ def test_simple_pipeline_no_offset_correction(debug):
         assert h5.attrs["experiment:sample"] == "data"
         assert h5.attrs["experiment:date"] == "2022-04-21"
         assert h5.attrs["experiment:run identifier"] == \
-               (f"d5a40aed-0b6c-0412-e87c-59789fdd28d0_"
-                f"dcn-{pp_hash[:7]}")
+            (f"d5a40aed-0b6c-0412-e87c-59789fdd28d0_"
+             f"dcn-{pp_hash[:7]}")
 
 
 def test_simple_pipeline_in_thread():