dclab 0.62.8__tar.gz → 0.62.10__tar.gz

{dclab-0.62.8 → dclab-0.62.10}/CHANGELOG

@@ -1,3 +1,14 @@
+0.62.10
+ - fix: `basin_definition_copy` failed when more than one basins defined
+   due to redundant loop
+ - fix: JSON logs during export were not sorted and indented
+0.62.9
+ - enh: sort basin definition before exporting to JSON; the side-effect
+   is that the unique key for a basin changes, possibly resulting in
+   duplicated basin definitions for existing datasets
+ - enh: make sure every basin can be identified with a key created from
+   its definition; important for connecting the basin dictionaries
+   with the instantiated basins
 0.62.8
  - fix: metadata of tables not copied by `copier.py`
  - docs: formatting typo (#270)

{dclab-0.62.8 → dclab-0.62.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dclab
-Version: 0.62.8
+Version: 0.62.10
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dclab-0.62.8 → dclab-0.62.10}/dclab/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.62.8'
-__version_tuple__ = version_tuple = (0, 62, 8)
+__version__ = version = '0.62.10'
+__version_tuple__ = version_tuple = (0, 62, 10)

{dclab-0.62.8 → dclab-0.62.10}/dclab/rtdc_dataset/copier.py

@@ -178,42 +178,51 @@ def basin_definition_copy(src_h5file, dst_h5file, features_iter):
     relevant for the internal basin.
     """
     dst_h5file.require_group("basins")
-    for b_key in src_h5file["basins"]:
+    # Load the basin information
+    basin_dicts = RTDC_HDF5.basin_get_dicts_from_h5file(src_h5file)
+    for bn in basin_dicts:
+        b_key = bn["key"]
+
         if b_key in dst_h5file["basins"]:
-            # This basin already exists.
+            # already stored therein
             continue
-        # Load the basin information
-        basin_dicts = RTDC_HDF5.basin_get_dicts_from_h5file(src_h5file)
-        for bn in basin_dicts:
-            if bn["type"] == "internal":
-                # Make sure we define the internal features selected
-                feat_used = [f for f in bn["features"] if f in features_iter]
-                if len(feat_used) == 0:
-                    # We don't have any internal features, don't write anything
-                    continue
-                elif feat_used != bn["features"]:
-                    bn["features"] = feat_used
-                    rewrite = True
-                else:
-                    rewrite = False
+
+        # sanity check
+        if b_key not in src_h5file["basins"]:
+            raise ValueError(
+                f"Failed to parse basin information correctly. Source file "
+                f"{src_h5file} does not contain basin {b_key} which I got "
+                f"from `RTDC_HDF5.basin_get_dicts_from_h5file`.")
+
+        if bn["type"] == "internal":
+            # Make sure we define the internal features selected
+            feat_used = [f for f in bn["features"] if f in features_iter]
+            if len(feat_used) == 0:
+                # We don't have any internal features, don't write anything
+                continue
+            elif feat_used != bn["features"]:
+                bn["features"] = feat_used
+                rewrite = True
             else:
-                # We do not have an internal basin, just copy everything
                 rewrite = False
+        else:
+            # We do not have an internal basin, just copy everything
+            rewrite = False
 
-            if rewrite:
-                # Convert edited `bn` to JSON and write feature data
-                b_lines = json.dumps(bn, indent=2).split("\n")
-                key = hashobj(b_lines)
-                if key not in dst_h5file["basins"]:
-                    with RTDCWriter(dst_h5file) as hw:
-                        hw.write_text(dst_h5file["basins"], key, b_lines)
-            else:
-                # copy only
-                h5ds_copy(src_loc=src_h5file["basins"],
-                          src_name=b_key,
-                          dst_loc=dst_h5file["basins"],
-                          dst_name=b_key,
-                          recursive=False)
+        if rewrite:
+            # Convert edited `bn` to JSON and write feature data
+            b_lines = json.dumps(bn, indent=2).split("\n")
+            key = hashobj(b_lines)
+            if key not in dst_h5file["basins"]:
+                with RTDCWriter(dst_h5file) as hw:
+                    hw.write_text(dst_h5file["basins"], key, b_lines)
+        else:
+            # copy only
+            h5ds_copy(src_loc=src_h5file["basins"],
+                      src_name=b_key,
+                      dst_loc=dst_h5file["basins"],
+                      dst_name=b_key,
+                      recursive=False)
 
 
 def h5ds_copy(src_loc, src_name, dst_loc, dst_name=None,

{dclab-0.62.8 → dclab-0.62.10}/dclab/rtdc_dataset/core.py

@@ -1,6 +1,7 @@
 """RT-DC dataset core classes and methods"""
 import abc
 import hashlib
+import json
 import os.path
 import pathlib
 import traceback
@@ -15,6 +16,7 @@ from .. import definitions as dfn
 from .. import downsampling
 from ..polygon_filter import PolygonFilter
 from .. import kde_methods
+from ..util import hashobj
 
 from .feat_anc_core import AncillaryFeature, FEATURES_RAPID
 from . import feat_basin
@@ -825,14 +827,20 @@ class RTDCBase(abc.ABC):
         # Sort basins according to priority
         bdicts_srt = sorted(self.basins_get_dicts(),
                             key=feat_basin.basin_priority_sorted_key)
-        bd_keys = [bd["key"] for bd in bdicts_srt if "key" in bd]
+        # complement basin "key"s (we do the same in writer)
+        for bdict in bdicts_srt:
+            if "key" not in bdict:
+                b_dat = json.dumps(bdict, indent=2, sort_keys=True).split("\n")
+                bdict["key"] = hashobj(b_dat)
+
+        bd_keys = [bd["key"] for bd in bdicts_srt]
         bd_keys += self._basins_ignored
         for bdict in bdicts_srt:
             if bdict["format"] not in bc:
                 warnings.warn(f"Encountered unsupported basin "
                               f"format '{bdict['format']}'!")
                 continue
-            if "key" in bdict and bdict["key"] in self._basins_ignored:
+            if bdict["key"] in self._basins_ignored:
                 warnings.warn(
                     f"Encountered cyclic basin dependency '{bdict['key']}'",
                     feat_basin.CyclicBasinDependencyFoundWarning)
@@ -853,6 +861,8 @@ class RTDCBase(abc.ABC):
                 "measurement_identifier": self.get_measurement_identifier(),
                 # allow to ignore basins
                 "ignored_basins": bd_keys,
+                # basin key
+                "key": bdict["key"],
             }
 
             # Check whether this basin is supported and exists

{dclab-0.62.8 → dclab-0.62.10}/dclab/rtdc_dataset/export.py

@@ -319,18 +319,21 @@ class Export(object):
 
             # write export log
             hw.store_log(time.strftime("dclab-export_%Y-%m-%d_%H.%M.%S"),
-                         json.dumps({
-                             "dclab version": version_tuple,
-                             "kwargs": {
-                                 "features": features,
-                                 "filtered": filtered,
-                                 "logs": logs,
-                                 "tables": tables,
-                                 "basins": basins,
-                                 "meta_prefix": meta_prefix,
-                                 "skip_checks": skip_checks
-                             }
-                         }).split("\n"))
+                         json.dumps(
+                             {"dclab version": version_tuple,
+                              "kwargs": {
+                                  "features": features,
+                                  "filtered": filtered,
+                                  "logs": logs,
+                                  "tables": tables,
+                                  "basins": basins,
+                                  "meta_prefix": meta_prefix,
+                                  "skip_checks": skip_checks
+                                  }
+                              },
+                             indent=2,
+                             sort_keys=True,
+                         ).split("\n"))
 
             if logs:
                 # write logs

{dclab-0.62.8 → dclab-0.62.10}/dclab/rtdc_dataset/feat_basin.py

@@ -9,6 +9,7 @@ import abc
 import numbers
 import threading
 from typing import Dict, List, Literal
+import uuid
 import warnings
 import weakref
 
@@ -74,6 +75,7 @@ class Basin(abc.ABC):
                                   ] = "same",
                  mapping_referrer: Dict = None,
                  ignored_basins: List[str] = None,
+                 key: str = None,
                  **kwargs):
         """
 
@@ -109,6 +111,10 @@ class Basin(abc.ABC):
             :class:`.RTDCBase`.
         ignored_basins: list of str
             List of basins to ignore in subsequent basin instantiations
+        key: str
+            Unique key to identify this basin; normally computed from
+            a JSON dump of the basin definition. A random string is used
+            if None is specified.
         kwargs:
             Additional keyword arguments passed to the `load_dataset`
             method of the `Basin` subclass.
@@ -124,6 +130,8 @@ class Basin(abc.ABC):
         self.name = name
         #: lengthy description of the basin
         self.description = description
+        # defining key of the basin
+        self.key = key or str(uuid.uuid4())
         # features this basin provides
         self._features = features
         #: measurement identifier of the referencing dataset
@@ -367,6 +375,7 @@ class BasinProxy:
             dataset to the downstream dataset
         """
         self.ds = ds
+        self.basins_get_dicts = ds.basins_get_dicts
         self.basinmap = basinmap
         self._features = {}
 

{dclab-0.62.8 → dclab-0.62.10}/dclab/rtdc_dataset/writer.py

@@ -389,7 +389,7 @@ class RTDCWriter:
                     flocs.append(str(pp.resolve()))
                     # Also store the relative path for user convenience.
                     # Don't use pathlib.Path.relative_to, because that
-                    # is deprecated in Python 3.12.
+                    # only has `walk_up` since Python 3.12.
                     # Also, just look in subdirectories which simplifies
                     # path resolution.
                     this_parent = str(self.path.parent) + os.sep
@@ -408,7 +408,7 @@ class RTDCWriter:
         else:
             raise ValueError(f"Unknown basin type '{basin_type}'")
 
-        b_lines = json.dumps(b_data, indent=2).split("\n")
+        b_lines = json.dumps(b_data, indent=2, sort_keys=True).split("\n")
         basins = self.h5file.require_group("basins")
         key = hashobj(b_lines)
         if key not in basins:

{dclab-0.62.8 → dclab-0.62.10}/dclab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dclab
-Version: 0.62.8
+Version: 0.62.10
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dclab-0.62.8 → dclab-0.62.10}/tests/test_cli_repack.py

@@ -80,6 +80,43 @@ def test_repack_basin_internal(use_basins):
             assert "userdef1" not in ds.features_basin
 
 
+@pytest.mark.filterwarnings(
+    "ignore::dclab.rtdc_dataset.config.WrongConfigurationTypeWarning")
+def test_repack_multiple_basins():
+    h5path = retrieve_data("fmt-hdf5_fl_wide-channel_2023.zip")
+    h5path_small = h5path.with_name("smaller.rtdc")
+    h5path_out = h5path.with_name("repacked.rtdc")
+
+    # Dataset creation
+    with h5py.File(h5path) as src, RTDCWriter(h5path_small) as hw:
+        # first, copy all the scalar features to the new file
+        rtdc_dataset.rtdc_copy(src_h5file=src,
+                               dst_h5file=hw.h5file,
+                               features="scalar")
+        hw.store_basin(basin_name="example basin",
+                       basin_type="file",
+                       basin_format="hdf5",
+                       basin_locs=[h5path],
+                       basin_descr="an example test basin",
+                       )
+
+        hw.store_basin(basin_name="example basin",
+                       basin_type="file",
+                       basin_format="hdf5",
+                       basin_locs=[h5path],
+                       basin_descr="a different test basin",
+                       )
+
+        assert len(hw.h5file["basins"].keys()) == 2
+
+    # repack the basin-based dataset
+    cli.repack(path_in=h5path_small,
+               path_out=h5path_out)
+
+    with new_dataset(h5path_out) as ds:
+        assert len(ds.basins) == 2
+
+
 def test_repack_no_data_from_basins_written():
     """
     When repacking a dataset, feature data from the basin should not be

{dclab-0.62.8 → dclab-0.62.10}/tests/test_rtdc_copier.py

@@ -203,6 +203,44 @@ def test_copy_basins_mapped():
         assert is_properly_compressed(hc["events/basinmap0"])
 
 
+def test_copy_basins_multiple():
+    path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
+    path_copy = path.with_name("test_copy.rtdc")
+
+    # add log to source file
+    with RTDCWriter(path, mode="append") as hw:
+        hw.store_basin(
+            basin_type="file",
+            basin_format="hdf5",
+            basin_name="test_basin",
+            basin_locs=["does-not-exist-but-does-not-matter.rtdc"],
+            basin_map=np.arange(len(hw.h5file["events/deform"])),
+            basin_descr="A first basin",
+            verify=False
+        )
+
+        hw.store_basin(
+            basin_type="file",
+            basin_format="hdf5",
+            basin_name="test_basin",
+            basin_locs=["does-not-exist-but-does-not-matter.rtdc"],
+            basin_map=np.arange(len(hw.h5file["events/deform"])),
+            basin_descr="A second basin",
+            verify=False
+        )
+
+        assert len(hw.h5file["basins"]) == 2
+
+    # copy
+    with h5py.File(path) as h5, h5py.File(path_copy, "w") as hc:
+        rtdc_copy(src_h5file=h5,
+                  dst_h5file=hc)
+
+    # Make sure this worked
+    with h5py.File(path_copy) as hc:
+        assert len(hc["basins"]) == 2
+
+
 def test_copy_basins_no_basin():
     path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
     path_copy = path.with_name("test_copy.rtdc")

{dclab-0.62.8 → dclab-0.62.10}/tests/test_rtdc_feat_basin.py

@@ -356,6 +356,68 @@ def test_basin_relative_paths(path_sep):
         assert "image" in ds
 
 
+@pytest.mark.filterwarnings(
+    "ignore::dclab.rtdc_dataset.config.WrongConfigurationTypeWarning")
+def test_basin_key_reproducible():
+    """Since 0.62.9 we sort basin keys"""
+    h5path = retrieve_data("fmt-hdf5_fl_wide-channel_2023.zip")
+    h5path_small = h5path.parent / "subdirectory" / "relative.rtdc"
+    h5path_small.parent.mkdir()
+    h5path_small_2 = h5path_small.with_name("from_writer.rtdc")
+
+    # Dataset creation
+    with h5py.File(h5path) as src, RTDCWriter(h5path_small) as hw:
+        # first, copy all the scalar features to the new file
+        rtdc_dataset.rtdc_copy(src_h5file=src,
+                               dst_h5file=hw.h5file,
+                               features="scalar")
+        # Next, store the basin information in the new dataset manually
+        bdat = {
+            "type": "file",
+            "format": "hdf5",
+            "features": ["image"],
+            "paths": [f"../{h5path.name}"]
+        }
+        blines = json.dumps(bdat, indent=2).split("\n")
+        basins = hw.h5file.require_group("basins")
+        hw.write_text(basins, "relative_paths_basin", blines)
+
+    # When loading a basin, dclab should use the existing key
+    # from the input file.
+    with dclab.new_dataset(h5path_small) as ds:
+        assert len(ds.basins) == 1
+        assert ds.basins[0].key == "relative_paths_basin"
+
+    # By default, dclab should store the basin with a key made from
+    # its hash.
+    # Dataset creation
+    with h5py.File(h5path) as src, RTDCWriter(h5path_small_2) as hw:
+        # first, copy all the scalar features to the new file
+        rtdc_dataset.rtdc_copy(src_h5file=src,
+                               dst_h5file=hw.h5file,
+                               features="scalar")
+        # Next, store the basin information in the new dataset with dclab
+        hw.store_basin(
+            basin_name="relative_paths_basin",
+            basin_type="file",
+            basin_format="hdf5",
+            basin_feats=["image"],
+            basin_locs=[f"../{h5path.name}"],
+            verify=False,
+        )
+    # When loading a basin, dclab should use the existing key
+    # from the input file.
+    with dclab.new_dataset(h5path_small_2) as ds:
+        assert len(ds.basins) == 1
+        # dclab 0.62.9
+        assert ds.basins[0].key in [
+            # posix linesep
+            "b8763fad05e17b58407875fe9c28d90b",
+            # win linesep
+            "a495fd2d495e0d991f3a4a313ec3a3fc",
+        ]
+
+
 def test_basin_sorting_basic():
     bnlist = [
         {"type": "remote", "format": "dcor", "ident": 0},