dcnum 0.25.10__tar.gz → 0.25.11__tar.gz

{dcnum-0.25.10 → dcnum-0.25.11}/CHANGELOG

@@ -1,3 +1,8 @@
+0.25.11
+ - enh: align measurement identifier computation with that of dclab
+ - enh: new `get_measurement_identifier` method
+ - enh: allow to optionally store measurement identifier in basin metadata
+ - enh: measurement identifier check for basin data
 0.25.10
  - fix: search for validation strings in logs did not use re.MULTILINE
  - enh: allow to validate torch model applicability based on meta values

{dcnum-0.25.10 → dcnum-0.25.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcnum
-Version: 0.25.10
+Version: 0.25.11
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -55,12 +55,12 @@ For more options, please check out the `documentation
 
 
 
-.. |dcnum| image:: https://raw.github.com/DC-analysis/dcnum/master/docs/logo/dcnum.png
+.. |dcnum| image:: https://raw.github.com/DC-analysis/dcnum/main/docs/logo/dcnum.png
 .. |PyPI Version| image:: https://img.shields.io/pypi/v/dcnum.svg
    :target: https://pypi.python.org/pypi/dcnum
 .. |Build Status| image:: https://img.shields.io/github/actions/workflow/status/DC-analysis/dcnum/check.yml
    :target: https://github.com/DC-analysis/dcnum/actions?query=workflow%3AChecks
-.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/DC-analysis/dcnum/master.svg
+.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/DC-analysis/dcnum
    :target: https://codecov.io/gh/DC-analysis/dcnum
 .. |Docs Status| image:: https://readthedocs.org/projects/dcnum/badge/?version=latest
    :target: https://readthedocs.org/projects/dcnum/builds/

{dcnum-0.25.10 → dcnum-0.25.11}/README.rst

@@ -24,12 +24,12 @@ For more options, please check out the `documentation
 
 
 
-.. |dcnum| image:: https://raw.github.com/DC-analysis/dcnum/master/docs/logo/dcnum.png
+.. |dcnum| image:: https://raw.github.com/DC-analysis/dcnum/main/docs/logo/dcnum.png
 .. |PyPI Version| image:: https://img.shields.io/pypi/v/dcnum.svg
    :target: https://pypi.python.org/pypi/dcnum
 .. |Build Status| image:: https://img.shields.io/github/actions/workflow/status/DC-analysis/dcnum/check.yml
    :target: https://github.com/DC-analysis/dcnum/actions?query=workflow%3AChecks
-.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/DC-analysis/dcnum/master.svg
+.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/DC-analysis/dcnum
    :target: https://codecov.io/gh/DC-analysis/dcnum
 .. |Docs Status| image:: https://readthedocs.org/projects/dcnum/badge/?version=latest
    :target: https://readthedocs.org/projects/dcnum/builds/

{dcnum-0.25.10 → dcnum-0.25.11}/src/dcnum/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.25.10'
-__version_tuple__ = version_tuple = (0, 25, 10)
+__version__ = version = '0.25.11'
+__version_tuple__ = version_tuple = (0, 25, 11)

{dcnum-0.25.10 → dcnum-0.25.11}/src/dcnum/logic/ctrl.py

@@ -1,6 +1,5 @@
 import collections
 import datetime
-import hashlib
 import importlib
 import json
 import logging
@@ -24,7 +23,7 @@ from ..feat import gate
 from ..feat import EventExtractorManagerThread
 from ..segm import SegmenterManagerThread, get_available_segmenters
 from ..meta import ppid
-from ..read import HDF5Data, get_mapping_indices
+from ..read import HDF5Data, get_measurement_identifier, get_mapping_indices
 from .._version import version, version_tuple
 from ..write import (
     DequeWriterThread, HDF5Writer, QueueCollectorThread, copy_features,
@@ -453,15 +452,7 @@ class DCNumJobRunner(threading.Thread):
                 # segmentation did actually take place.
                 mid_ap = f"dcn-{self.pphash[:7]}"
                 # This is the current measurement identifier
-                mid_cur = hw.h5.attrs.get("experiment:run identifier")
-                if not mid_cur:
-                    # Compute a measurement identifier from the metadata
-                    m_time = hw.h5.attrs.get("experiment:time", "none")
-                    m_date = hw.h5.attrs.get("experiment:date", "none")
-                    m_sid = hw.h5.attrs.get("setup:identifier", "none")
-                    hasher = hashlib.md5(
-                        f"{m_time}_{m_date}_{m_sid}".encode("utf-8"))
-                    mid_cur = str(uuid.UUID(hex=hasher.hexdigest()))
+                mid_cur = get_measurement_identifier(hw.h5)
                 # The new measurement identifier is a combination of both.
                 mid_new = f"{mid_cur}_{mid_ap}" if mid_cur else mid_ap
                 hw.h5.attrs["experiment:run identifier"] = mid_new
@@ -666,6 +657,7 @@ class DCNumJobRunner(threading.Thread):
                                    mapping=basinmap0,
                                    paths=paths,
                                    description=f"Created with dcnum {version}",
+                                   identifier=get_measurement_identifier(hin),
                                    )
                 self._progress_bn += 1 / len(feats_raw)
         t_tot = time.perf_counter() - t0

{dcnum-0.25.10 → dcnum-0.25.11}/src/dcnum/read/__init__.py

@@ -2,6 +2,9 @@
 from .cache import md5sum
 from .const import PROTECTED_FEATURES
 from .detect_flicker import detect_flickering
-from .hdf5_data import HDF5Data, HDF5ImageCache
+from .hdf5_data import (
+    HDF5Data, HDF5ImageCache, get_measurement_identifier,
+    BasinIdentifierMismatchError
+)
 from .hdf5_concat import concatenated_hdf5_data
 from .mapped import get_mapping_indices, get_mapped_object

{dcnum-0.25.10 → dcnum-0.25.11}/src/dcnum/read/hdf5_data.py

@@ -16,6 +16,10 @@ from .const import PROTECTED_FEATURES
 from .mapped import get_mapped_object, get_mapping_indices
 
 
+class BasinIdentifierMismatchError(BaseException):
+    """Used when basin identifiers do not match"""
+
+
 class HDF5Data:
     """HDF5 (.rtdc) input file data instance"""
     def __init__(self,
@@ -473,6 +477,12 @@ class HDF5Data:
             features = []
         else:
             h5 = h5py.File(path, "r")
+            # verify that the basin was identified correctly
+            if ((id_exp := bn_dict.get("identifier")) is not None
+                    and (id_act := get_measurement_identifier(h5)) != id_exp):
+                raise BasinIdentifierMismatchError(
+                    f"The basin '{path}' with identifier '{id_act}' "
+                    f"does not match the expected identifier '{id_exp}'")
             h5group = h5["events"]
             # features defined in the basin
             features = bn_dict.get("features")
@@ -560,3 +570,30 @@ def concatenated_hdf5_data(*args, **kwargs):
         DeprecationWarning)
     from . import hdf5_concat
     return hdf5_concat.concatenated_hdf5_data(*args, **kwargs)
+
+
+def get_measurement_identifier(h5: h5py.Group) -> str | None:
+    """Return the measurement identifier for the given H5File object
+
+    The basin identifier is taken from the HDF5 attributes. If the
+    "experiment:run identifier" attribute is not set, it is
+    computed from the HDF5 attributes "experiment:time",
+    "experiment:date", and "setup:identifier".
+
+    If the measurement identifier cannot be found or computed,
+    return None.
+    """
+    # This is the current measurement identifier
+    mid = h5.attrs.get("experiment:run identifier")
+    if not mid:
+        # Compute a measurement identifier from the metadata
+        m_time = h5.attrs.get("experiment:time", None) or None
+        m_date = h5.attrs.get("experiment:date", None) or None
+        m_sid = h5.attrs.get("setup:identifier", None) or None
+        if None not in [m_time, m_date, m_sid]:
+            # Only compute an identifier if all of the above
+            # are defined.
+            hasher = hashlib.md5(
+                f"{m_time}_{m_date}_{m_sid}".encode("utf-8"))
+            mid = str(uuid.UUID(hex=hasher.hexdigest()))
+    return mid

{dcnum-0.25.10 → dcnum-0.25.11}/src/dcnum/write/writer.py

@@ -8,7 +8,7 @@ import h5py
 import hdf5plugin
 import numpy as np
 
-from ..read import HDF5Data
+from ..read import HDF5Data, get_measurement_identifier
 from .._version import version
 
 
@@ -157,6 +157,7 @@ class HDF5Writer:
                     description: str | None = None,
                     mapping: np.ndarray = None,
                     internal_data: Dict | None = None,
+                    identifier: str | None = None,
                     ):
         """Write an HDF5-based file basin
 
@@ -177,6 +178,10 @@ class HDF5Writer:
         internal_data: dict of ndarrays
             internal basin data to store; If this is set, then `features`
             and `paths` must be set to `None`.
+        identifier: str
+            the measurement identifier of the basin as computed by
+            the :func:`~dcnum.read.hdf5_data.get_measurement_identifier`
+            function.
         """
         bdat = {
             "description": description,
@@ -190,6 +195,9 @@ class HDF5Writer:
             if paths is not None:
                 raise ValueError("`paths` must be set to None when storing "
                                  "internal basin features")
+            if identifier is not None:
+                warnings.warn(f"Not storing identifier for internal "
+                              f"basin '{name}' (got '{identifier}')")
             # store the internal basin information
             for feat in internal_data:
                 if feat in self.h5.require_group("basin_events"):
@@ -206,6 +214,8 @@ class HDF5Writer:
             bdat["format"] = "hdf5"
             bdat["paths"] = [str(pp) for pp in paths]
             bdat["type"] = "file"
+            # identifier only makes sense here (not for internal basins)
+            bdat["identifier"] = identifier
 
         # Explicit features stored in basin file
         if features is not None and len(features):
@@ -322,6 +332,7 @@ def create_with_basins(
         warnings.warn(f"Creating basin-based file '{path_out}' without any "
                       f"basins, since the list `basin_paths' is empty!",
                       CreatingFileWithoutBasinWarning)
+        basin_paths = []
     with HDF5Writer(path_out, mode="w") as hw:
         # Get the metadata from the first available basin path
 
@@ -359,16 +370,19 @@ def create_with_basins(
                     features = sorted(h5["events"].keys())
                     features = [f for f in features if
                                 not f.startswith("basinmap")]
+                    basin_identifier = get_measurement_identifier(h5)
                 name = prep.name
             else:
                 features = None
                 name = bps[0]
+                basin_identifier = None
 
             # Write the basin data
             hw.store_basin(name=name,
                            paths=bps,
                            features=features,
                            description=f"Created by dcnum {version}",
+                           identifier=basin_identifier,
                            )
 
 
@@ -404,6 +418,7 @@ def copy_basins(h5_src: h5py.File,
                            paths=bn_dict["paths"],
                            features=bn_dict["features"],
                            mapping=mapping,
+                           identifier=bn_dict.get("identifier"),
                            )
         else:
             warnings.warn(f"Ignored basin of type '{bn_dict['type']}'",

{dcnum-0.25.10 → dcnum-0.25.11}/src/dcnum.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcnum
-Version: 0.25.10
+Version: 0.25.11
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -55,12 +55,12 @@ For more options, please check out the `documentation
 
 
 
-.. |dcnum| image:: https://raw.github.com/DC-analysis/dcnum/master/docs/logo/dcnum.png
+.. |dcnum| image:: https://raw.github.com/DC-analysis/dcnum/main/docs/logo/dcnum.png
 .. |PyPI Version| image:: https://img.shields.io/pypi/v/dcnum.svg
    :target: https://pypi.python.org/pypi/dcnum
 .. |Build Status| image:: https://img.shields.io/github/actions/workflow/status/DC-analysis/dcnum/check.yml
    :target: https://github.com/DC-analysis/dcnum/actions?query=workflow%3AChecks
-.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/DC-analysis/dcnum/master.svg
+.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/DC-analysis/dcnum
    :target: https://codecov.io/gh/DC-analysis/dcnum
 .. |Docs Status| image:: https://readthedocs.org/projects/dcnum/badge/?version=latest
    :target: https://readthedocs.org/projects/dcnum/builds/

{dcnum-0.25.10 → dcnum-0.25.11}/tests/test_logic_pipeline.py

@@ -41,11 +41,10 @@ def get_log(hd: read.hdf5_data.HDF5Data,
       },
      ["experiment:run identifier"],
      "f0cfdc6f-93c7-c093-d135-a20f3e5bdbfa"],
-    # delete everything, this is the hash of none-none-none
+    # delete everything, this should yield only the pipeline ID
     [{},
-     ["setup:identifier", "experiment:time", "experiment:run identifier",
-      "setup:identifier"],
-     "07d18822-3045-4e6b-da0b-f1a16c80132a"],
+     ["setup:identifier", "experiment:time", "experiment:run identifier"],
+     None],
 ])
 def test_basin_experiment_identifier_correct(
         override_attrs, delete_keys, meas_id):
@@ -77,7 +76,10 @@ def test_basin_experiment_identifier_correct(
 
     with h5py.File(path_out) as hout:
         appid = hout.attrs["pipeline:dcnum hash"]
-        eri_exp = f"{meas_id}_dcn-{appid[:7]}"
+        if meas_id is not None:
+            eri_exp = f"{meas_id}_dcn-{appid[:7]}"
+        else:
+            eri_exp = f"dcn-{appid[:7]}"
         # this is the actual test
         assert hout.attrs["experiment:run identifier"] == eri_exp
 
@@ -185,9 +187,23 @@ def test_basin_strategy_tap():
         assert "image" not in h5["events"]
         for feat in h5["events"]:
             assert len(h5["events"][feat]) == 275
+        # Make sure the correct basin identifier is stored
+        # (must be the original identifier plus a dcn pipeline hash tag)
+        assert h5.attrs["experiment:run identifier"] \
+            == "d5a40aed-0b6c-0412-e87c-59789fdd28d0_dcn-17decbe"
+
         # The other features are accessed via basins
         hd = read.HDF5Data(h5)
         assert "image" in hd
+        # Check whether the basin identifier is set correctly.
+        for bn in hd.basins:
+            print(bn)
+            if bn["type"] == "file":
+                assert bn["identifier"] \
+                   == "d5a40aed-0b6c-0412-e87c-59789fdd28d0"
+                break
+        else:
+            assert False, "Something went wrong, the basin is missing"
 
 
 def test_basin_strategy_tap_rollmed():

{dcnum-0.25.10 → dcnum-0.25.11}/tests/test_read_hdf5_basins.py

@@ -76,6 +76,67 @@ def test_basin_features_path_absolute_mapped():
         assert np.allclose(hd["image"][:], hd_src["image"][4:10])
 
 
+def test_basin_identifier_mismatch(tmp_path):
+    path = retrieve_data(
+        "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+    path_test = path.parent / "test.h5"
+
+    with h5py.File(path, "a") as h5:
+        rid = read.get_measurement_identifier(h5)
+        area_um = h5["events/area_um"][:]
+        del h5["events/area_um"]
+        assert rid == "d5a40aed-0b6c-0412-e87c-59789fdd28d0"
+
+    # We basically create a file that consists only of the metadata.
+    with write.HDF5Writer(path_test) as hw:
+        hw.store_basin(name="pidemon",
+                       paths=[path],
+                       features=["deform"],
+                       description="Wrong basin identifier specified",
+                       identifier=rid + "-wrong",
+                       )
+        hw.store_feature_chunk("area_um", area_um)
+
+    with h5py.File(path_test) as hb:
+        basins = read.HDF5Data.extract_basin_dicts(hb)
+        assert len(basins) == 1
+
+    with read.HDF5Data(path_test) as hd:
+        # Accessing data in the new file works
+        assert hd["area_um"][0]
+        with pytest.raises(read.BasinIdentifierMismatchError,
+                           match="does not match"):
+            hd["deform"][0]
+
+
+def test_basin_identifier_normal_use_case(tmp_path):
+    path = retrieve_data(
+        "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+    path_test = path.parent / "test.h5"
+
+    with h5py.File(path) as h5:
+        rid = read.get_measurement_identifier(h5)
+        assert rid == "d5a40aed-0b6c-0412-e87c-59789fdd28d0"
+
+    # We basically create a file that consists only of the metadata.
+    with write.HDF5Writer(path_test) as hw:
+        hw.store_basin(name="pidemon",
+                       paths=[path],
+                       description="Basin identifier specified",
+                       identifier=rid,
+                       )
+
+    with h5py.File(path_test) as hb:
+        basins = read.HDF5Data.extract_basin_dicts(hb)
+        assert len(basins) == 1
+        bn = basins[0]
+        assert bn["identifier"] == rid
+
+    with read.HDF5Data(path_test) as hd:
+        # Accessing data in the new file works
+        assert hd["deform"][0]
+
+
 def test_basin_mapped_with_mapped_dataset():
     """You can have a mapped dataset opening a mapped basin"""
     path_src = retrieve_data(