dclab 0.62.6__tar.gz → 0.62.8__tar.gz

{dclab-0.62.6 → dclab-0.62.8}/CHANGELOG

@@ -1,3 +1,14 @@
+0.62.8
+ - fix: metadata of tables not copied by `copier.py`
+ - docs: formatting typo (#270)
+0.62.7
+ - fix: IntegrityChecker must not load basins
+ - fix: KeyError in IntegrityChecker when "basin_events" group missing (#268)
+ - fix: warn user about missing internal basin features while still loading
+   those that are available (#268)
+ - fix: do not copy internal basin definitions for non-existent features (#269)
+ - docs: clarify interpretation of [qpi]: "filter size" metadata (#267)
+ - tests: avoid architecture and Python-version dependent test
 0.62.6
  - enh: support passing pathlib.Path to lme4.rsetup.set_*_path methods
  - docs: correct keyword arguments for advanced section on S3
@@ -17,7 +28,8 @@
 0.62.0
  - BREAKING: removed the `rtdc_dataset.linker` module, because it is not
    used by any other piece of software and because combining file-like
-   objects external links are quirky and their combination finally broke with h5py 3.12
+   objects external links are quirky and their combination finally broke
+   with h5py 3.12
  - enh: added integrity check for external links
  - tests: reduce warnings
 0.61.5

{dclab-0.62.6 → dclab-0.62.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dclab
-Version: 0.62.6
+Version: 0.62.8
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dclab-0.62.6 → dclab-0.62.8}/dclab/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.62.6'
-__version_tuple__ = version_tuple = (0, 62, 6)
+__version__ = version = '0.62.8'
+__version_tuple__ = version_tuple = (0, 62, 8)

{dclab-0.62.6 → dclab-0.62.8}/dclab/definitions/meta_const.py

@@ -161,7 +161,17 @@ CFG_METADATA = {
         ["subtract mean", fbool, "Subtract mean before processing"],
         # pipeline_kws
         ["filter name", str, "Fourier filter used"],
-        # Corresponds to `filter_size_interpretation="frequency index"`
+        # qpretrieve defines the keyword argument `filter_size_interpretation`
+        # for determining the filter size in Fourier space. In DC, we
+        # need a well-defined value for the filter size. The most logical
+        # choice is to interpret the filter size as "frequency index", which
+        # is independent of the image shape and yields a good approximation
+        # of the actual resolution one can expect. The default value
+        # ("sideband distance") is a good choice for general QPI analysis,
+        # but there is no meaningful information one could extract from it
+        # by just looking at the number. Thus, the "filter size" that we
+        # see here corresponds to a filter size set in qpretrieve where
+        # `filter_size_interpretation="frequency index"`.
         ["filter size", float, "Fourier filter size [1/pix]"],
         ["scale to filter", fboolorfloat, "Scale QPI data to filter size"],
         # x, y coordinates, don't set if you wish None to be the default

{dclab-0.62.6 → dclab-0.62.8}/dclab/rtdc_dataset/check.py

@@ -193,7 +193,7 @@ class IntegrityChecker(object):
         else:
             with warnings.catch_warnings(record=True) as ws:
                 warnings.simplefilter("always")
-                self.ds = load_file(path_or_ds)
+                self.ds = load_file(path_or_ds, enable_basins=False)
                 for ww in ws:
                     self.warn_cues.append(ICue(
                         msg=f"{ww.category.__name__}: {ww.message}",
@@ -260,13 +260,24 @@ class IntegrityChecker(object):
                              level="alert",
                              category="basin data",
                              ))
-                for feat in bn["features"]:
-                    if feat not in self.ds.h5file[bpaths[0]]:
+                else:
+                    if "basin_events" not in self.ds.h5file:
                         cues.append(
-                            ICue(msg=f"Missing internal basin feature {feat}",
+                            ICue(msg="Missing internal basin group "
+                                     "'basin_events', although an internal "
+                                     "basin is defined",
                                  level="violation",
                                  category="basin data",
                                  ))
+                    else:
+                        for feat in bn["features"]:
+                            if feat not in self.ds.h5file["basin_events"]:
+                                cues.append(
+                                    ICue(msg=f"Missing internal basin "
+                                             f"feature {feat}",
+                                         level="violation",
+                                         category="basin data",
+                                         ))
         return cues
 
     def check_compression(self, **kwargs):

{dclab-0.62.6 → dclab-0.62.8}/dclab/rtdc_dataset/copier.py

@@ -1,6 +1,7 @@
 """Helper methods for copying .rtdc data"""
 from __future__ import annotations
 
+import json
 import re
 from typing import List, Literal
 
@@ -10,8 +11,10 @@ import hdf5plugin
 import numpy as np
 
 from ..definitions import feature_exists, scalar_feature_exists
+from ..util import hashobj
 
-from .fmt_hdf5 import DEFECTIVE_FEATURES
+from .fmt_hdf5 import DEFECTIVE_FEATURES, RTDC_HDF5
+from .writer import RTDCWriter
 
 
 def rtdc_copy(src_h5file: h5py.Group,
@@ -44,8 +47,7 @@ def rtdc_copy(src_h5file: h5py.Group,
         Add this prefix to the name of the logs and tables in `dst_h5file`.
     """
     # metadata
-    for akey in src_h5file.attrs:
-        dst_h5file.attrs[akey] = src_h5file.attrs[akey]
+    dst_h5file.attrs.update(src_h5file.attrs)
 
     # events in source file
     if "events" in src_h5file:
@@ -57,19 +59,6 @@ def rtdc_copy(src_h5file: h5py.Group,
         events_src += list(src_h5file["basin_events"].keys())
         events_src = sorted(set(events_src))
 
-    # basins
-    if include_basins and "basins" in src_h5file:
-        dst_h5file.require_group("basins")
-        for b_key in src_h5file["basins"]:
-            if b_key in dst_h5file["basins"]:
-                # This basin already exists.
-                continue
-            h5ds_copy(src_loc=src_h5file["basins"],
-                      src_name=b_key,
-                      dst_loc=dst_h5file["basins"],
-                      dst_name=b_key,
-                      recursive=False)
-
     # logs
     if include_logs and "logs" in src_h5file:
         dst_h5file.require_group("logs")
@@ -94,11 +83,12 @@ def rtdc_copy(src_h5file: h5py.Group,
             #           dst_loc=dst_h5file["tables"],
             #           dst_name=meta_prefix + tkey,
             #           recursive=False)
-            dst_h5file["tables"].create_dataset(
+            copy_table = dst_h5file["tables"].create_dataset(
                 name=tkey,
                 data=src_h5file["tables"][tkey][:],
                 fletcher32=True,
                 **hdf5plugin.Zstd(clevel=5))
+            copy_table.attrs.update(src_h5file["tables"][tkey].attrs)
 
     # events
     if isinstance(features, list):
@@ -130,6 +120,12 @@ def rtdc_copy(src_h5file: h5py.Group,
             if feat in feature_iter:
                 feature_iter.remove(feat)
 
+    # copy basin definitions
+    if include_basins and "basins" in src_h5file:
+        basin_definition_copy(src_h5file=src_h5file,
+                              dst_h5file=dst_h5file,
+                              features_iter=feature_iter)
+
     if feature_iter:
         dst_h5file.require_group("events")
         for feat in feature_iter:
@@ -170,6 +166,56 @@ def rtdc_copy(src_h5file: h5py.Group,
                               )
 
 
+def basin_definition_copy(src_h5file, dst_h5file, features_iter):
+    """Copy basin definitions `src_h5file["basins"]` to the new file
+
+    Normally, we would just use :func:`h5ds_copy` to copy basins from
+    one dataset to another. However, if we are e.g. only copying scalar
+    features, and there are non-scalar features in the internal basin,
+    then we must rewrite the basin definition of the internal basin.
+
+    The `features_iter` list of features defines which features are
+    relevant for the internal basin.
+    """
+    dst_h5file.require_group("basins")
+    for b_key in src_h5file["basins"]:
+        if b_key in dst_h5file["basins"]:
+            # This basin already exists.
+            continue
+        # Load the basin information
+        basin_dicts = RTDC_HDF5.basin_get_dicts_from_h5file(src_h5file)
+        for bn in basin_dicts:
+            if bn["type"] == "internal":
+                # Make sure we define the internal features selected
+                feat_used = [f for f in bn["features"] if f in features_iter]
+                if len(feat_used) == 0:
+                    # We don't have any internal features, don't write anything
+                    continue
+                elif feat_used != bn["features"]:
+                    bn["features"] = feat_used
+                    rewrite = True
+                else:
+                    rewrite = False
+            else:
+                # We do not have an internal basin, just copy everything
+                rewrite = False
+
+            if rewrite:
+                # Convert edited `bn` to JSON and write feature data
+                b_lines = json.dumps(bn, indent=2).split("\n")
+                key = hashobj(b_lines)
+                if key not in dst_h5file["basins"]:
+                    with RTDCWriter(dst_h5file) as hw:
+                        hw.write_text(dst_h5file["basins"], key, b_lines)
+            else:
+                # copy only
+                h5ds_copy(src_loc=src_h5file["basins"],
+                          src_name=b_key,
+                          dst_loc=dst_h5file["basins"],
+                          dst_name=b_key,
+                          recursive=False)
+
+
 def h5ds_copy(src_loc, src_name, dst_loc, dst_name=None,
               ensure_compression=True, recursive=True):
     """Copy an HDF5 Dataset from one group to another
@@ -257,8 +303,7 @@ def h5ds_copy(src_loc, src_name, dst_loc, dst_name=None,
                 for chunk in src.iter_chunks():
                     dst[chunk] = src[chunk]
             # Also write all the attributes
-            for key in src.attrs:
-                dst.attrs[key] = src.attrs[key]
+            dst.attrs.update(src.attrs)
         else:
             # Copy the Dataset to the destination as-is.
             h5py.h5o.copy(src_loc=src_loc.id,

{dclab-0.62.6 → dclab-0.62.8}/dclab/rtdc_dataset/export.py

@@ -224,6 +224,7 @@ class Export(object):
         skip_checks: bool
             Disable checking whether all features have the same length.
 
+
         .. versionchanged:: 0.58.0
 
            The ``basins`` keyword argument was added, and it is now possible

{dclab-0.62.6 → dclab-0.62.8}/dclab/rtdc_dataset/feat_basin.py

@@ -17,6 +17,10 @@ import numpy as np
 from ..util import copy_if_needed
 
 
+class BasinFeatureMissingWarning(UserWarning):
+    """Used when a badin feature is defined but not stored"""
+
+
 class CyclicBasinDependencyFoundWarning(UserWarning):
     """Used when a basin is defined in one of its sub-basins"""
 
@@ -498,6 +502,19 @@ class InternalH5DatasetBasin(Basin):
         if self._features is None:
             raise ValueError("You must specify features when defining "
                              "internal basins.")
+        # Redefine the features if necessary
+        h5root = self._basinmap_referrer().h5file
+        available_features = []
+        for feat in self._features:
+            if self.location in h5root and feat in h5root[self.location]:
+                available_features.append(feat)
+            else:
+                warnings.warn(
+                    f"Feature '{feat}' is defined as an internal basin, "
+                    f"but it cannot be found in '{self.location}'.",
+                    BasinFeatureMissingWarning)
+        self._features.clear()
+        self._features += available_features
 
     def _load_dataset(self, location, **kwargs):
         from .fmt_dict import RTDC_Dict
@@ -509,8 +526,7 @@ class InternalH5DatasetBasin(Basin):
         return RTDC_Dict(ds_dict)
 
     def is_available(self):
-        h5root = self._basinmap_referrer().h5file
-        return self.location in h5root
+        return bool(self._features)
 
     def verify_basin(self, *args, **kwargs):
         """It's not necessary to verify internal basins"""

{dclab-0.62.6 → dclab-0.62.8}/dclab/rtdc_dataset/fmt_hdf5/base.py

@@ -175,10 +175,15 @@ class RTDC_HDF5(RTDCBase):
 
     def basins_get_dicts(self):
         """Return list of dicts for all basins defined in `self.h5file`"""
+        return self.basin_get_dicts_from_h5file(self.h5file)
+
+    @staticmethod
+    def basin_get_dicts_from_h5file(h5file):
+        """Return list of dicts for all basins defined in `h5file`"""
         basins = []
         # Do not sort anything here, sorting is done in `RTDCBase`.
-        for bk in self.h5file.get("basins", []):
-            bdat = list(self.h5file["basins"][bk])
+        for bk in h5file.get("basins", []):
+            bdat = list(h5file["basins"][bk])
             if isinstance(bdat[0], bytes):
                 bdat = [bi.decode("utf") for bi in bdat]
             bdict = json.loads(" ".join(bdat))

{dclab-0.62.6 → dclab-0.62.8}/dclab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dclab
-Version: 0.62.6
+Version: 0.62.8
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dclab-0.62.6 → dclab-0.62.8}/tests/test_cli_condense.py

@@ -134,10 +134,12 @@ def test_condense_basins_include_no_data_from_basins():
 @pytest.mark.parametrize("store_basins", [True, False])
 def test_condense_basins_internal(store_basins):
     """
-    Internal basins should just be copied to the new file
+    Internal basins should just be copied to the new file.
+    However, image data are not scalar data, so they should not
+    show up in the condensed file.
     """
     h5path = retrieve_data("fmt-hdf5_fl_wide-channel_2023.zip")
-    h5path_small = h5path.with_name("smaller.rtdc")
+    h5path_small = h5path.with_name("input_file.rtdc")
     h5path_out = h5path.with_name("condensed.rtdc")
 
     # Dataset creation
@@ -146,7 +148,9 @@ def test_condense_basins_internal(store_basins):
         rtdc_dataset.rtdc_copy(src_h5file=src,
                                dst_h5file=hw.h5file,
                                features="scalar")
-        hw.store_basin(basin_name="example basin",
+        assert "basins" not in hw.h5file, "no basins in input file"
+        # store scalar and non-scalar internal basins in the input file
+        hw.store_basin(basin_name="scalar and non-scalar basin data",
                        basin_type="internal",
                        basin_format="h5dataset",
                        basin_locs=["basin_events"],
@@ -165,8 +169,9 @@ def test_condense_basins_internal(store_basins):
         assert "image_bg" in ds.features_basin
         assert "image_bg" not in ds.features_innate
 
-    # compress the basin-based dataset
-    cli.condense(path_in=h5path_small, path_out=h5path_out,
+    # condense the basin-based dataset
+    cli.condense(path_in=h5path_small,
+                 path_out=h5path_out,
                  store_basin_features=store_basins)
 
     with h5py.File(h5path_out) as h5:
@@ -182,8 +187,10 @@ def test_condense_basins_internal(store_basins):
         assert "image_bg" not in h5["basin_events"]
 
     with new_dataset(h5path_out) as ds:
+        # userdef1 is an internal, scalar basin
         assert "userdef1" in ds.features_basin
-        assert "image_bg" in ds.features_basin
+        # image_bg is not scalar, so it should not be here
+        assert "image_bg" not in ds.features_basin
 
 
 @pytest.mark.filterwarnings(

{dclab-0.62.6 → dclab-0.62.8}/tests/test_rtdc_check_dataset.py

@@ -48,7 +48,13 @@ def test_basic():
     assert "Compression: None" in info
 
 
-def test_basin_features_internal_missing_feature():
+@pytest.mark.parametrize("h5_obj_delete", [
+    # delete the entire basin_events directory
+    "basin_events",
+    # delete only the feature data
+    "basin_events/userdef1"
+])
+def test_basin_features_internal_missing_feature(h5_obj_delete):
     path = retrieve_data("fmt-hdf5_polygon_gate_2021.zip")
     with RTDCWriter(path) as hw:
         size = hw.h5file["events/deform"].shape[0]
@@ -61,7 +67,7 @@ def test_basin_features_internal_missing_feature():
             basin_locs=["basin_events"],
             internal_data={"userdef1": np.arange(2)},
         )
-        del hw.h5file["basin_events/userdef1"]
+        del hw.h5file[h5_obj_delete]
 
     # see if we can open the file without any error
     with check.IntegrityChecker(path) as ic:

{dclab-0.62.6 → dclab-0.62.8}/tests/test_rtdc_copier.py

@@ -67,6 +67,113 @@ def test_copy_basins():
         assert is_properly_compressed(hc[f"basins/{bn_hash}"])
 
 
+def test_copy_basins_internal_none():
+    """Store an internal basin and only export scalar features"""
+    h5path = retrieve_data("fmt-hdf5_fl_wide-channel_2023.zip")
+    h5path_small = h5path.with_name("input_file.rtdc")
+    h5path_out = h5path.with_name("condensed.rtdc")
+
+    # Dataset creation
+    with h5py.File(h5path) as src, RTDCWriter(h5path_small) as hw:
+        # first, copy all the scalar features to the new file
+        rtdc_copy(src_h5file=src,
+                  dst_h5file=hw.h5file,
+                  features="scalar")
+        assert "basins" not in hw.h5file, "no basins in input file"
+        # store scalar and non-scalar internal basins in the input file
+        hw.store_basin(basin_name="scalar and non-scalar basin data",
+                       basin_type="internal",
+                       basin_format="h5dataset",
+                       basin_locs=["basin_events"],
+                       basin_descr="an example test basin",
+                       internal_data={"image_bg": np.zeros((2, 80, 320))},
+                       basin_map=np.zeros(src["events/deform"].shape[0]),
+                       basin_feats=["image_bg"],
+                       )
+
+    # sanity check
+    with new_dataset(h5path_small) as ds:
+        assert "image_bg" in ds.features_basin
+        assert "image_bg" not in ds.features_innate
+
+    # Now the actual tests starts.
+    with h5py.File(h5path_small) as src, h5py.File(h5path_out, "a") as dst:
+        rtdc_copy(src_h5file=src,
+                  dst_h5file=dst,
+                  features="scalar",
+                  include_basins=True,
+                  include_logs=True,
+                  include_tables=True,
+                  meta_prefix="")
+
+    with h5py.File(h5path_out) as h5:
+        # The output file should not contain the "image_bg" data.
+        assert "basin_events" not in h5
+
+    # When we load the basin data, "image_bg" should also not be there.
+    with dclab.new_dataset(h5path_out) as ds:
+        basin_dicts = ds.basins_get_dicts()
+        assert len(basin_dicts) == 0
+
+
+def test_copy_basins_internal_no_scalar():
+    """Store an internal basin and only export scalar features"""
+    h5path = retrieve_data("fmt-hdf5_fl_wide-channel_2023.zip")
+    h5path_small = h5path.with_name("input_file.rtdc")
+    h5path_out = h5path.with_name("condensed.rtdc")
+
+    # Dataset creation
+    with h5py.File(h5path) as src, RTDCWriter(h5path_small) as hw:
+        # first, copy all the scalar features to the new file
+        rtdc_copy(src_h5file=src,
+                  dst_h5file=hw.h5file,
+                  features="scalar")
+        assert "basins" not in hw.h5file, "no basins in input file"
+        # store scalar and non-scalar internal basins in the input file
+        hw.store_basin(basin_name="scalar and non-scalar basin data",
+                       basin_type="internal",
+                       basin_format="h5dataset",
+                       basin_locs=["basin_events"],
+                       basin_descr="an example test basin",
+                       internal_data={"userdef1": np.arange(2),
+                                      "image_bg": np.zeros((2, 80, 320)),
+                                      },
+                       basin_map=np.zeros(src["events/deform"].shape[0]),
+                       basin_feats=["image_bg", "userdef1"],
+                       )
+
+    # sanity check
+    with new_dataset(h5path_small) as ds:
+        assert "userdef1" in ds.features_basin
+        assert "userdef1" not in ds.features_innate
+        assert "image_bg" in ds.features_basin
+        assert "image_bg" not in ds.features_innate
+
+    # Now the actual tests starts.
+    with h5py.File(h5path_small) as src, h5py.File(h5path_out, "a") as dst:
+        rtdc_copy(src_h5file=src,
+                  dst_h5file=dst,
+                  features="scalar",
+                  include_basins=True,
+                  include_logs=True,
+                  include_tables=True,
+                  meta_prefix="")
+
+    with h5py.File(h5path_out) as h5:
+        # The output file should not contain the "image_bg" data.
+        assert "userdef1" in h5["basin_events"]
+        assert "image_bg" not in h5["basin_events"]
+
+    # When we load the basin data, "image_bg" should also not be there.
+    with dclab.new_dataset(h5path_out) as ds:
+        basin_dicts = ds.basins_get_dicts()
+        assert len(basin_dicts) == 1
+        bn = basin_dicts[0]
+        assert bn["type"] == "internal"
+        assert bn["mapping"] != "same"
+        assert bn["features"] == ["userdef1"], "image_bg feature must be gone"
+
+
 def test_copy_basins_mapped():
     path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
     path_copy = path.with_name("test_copy.rtdc")
@@ -326,6 +433,11 @@ def test_copy_tables_hdf5_issue_3214():
         rtdc_copy(src_h5file=h5,
                   dst_h5file=hc)
 
+    # Also make sure metadata are copied
+    with h5py.File(path_copy) as hc:
+        assert hc["tables"]["cytoshot_monitor"].attrs["COLOR_shift"]\
+               == "#0e8f69"
+
 
 def test_copy_with_compression():
     path = retrieve_data("fmt-hdf5_image-bg_2020.zip")

{dclab-0.62.6 → dclab-0.62.8}/tests/test_rtdc_feat_anc_plugin.py

@@ -1,5 +1,4 @@
 import pathlib
-import sys
 import tempfile
 import traceback
 
@@ -134,21 +133,15 @@ def test_pf_attribute_ancill_info():
 def test_pf_attribute_plugin_feature_info():
     """Check the plugin feature info attribute"""
     info = example_plugin_info_single_feature()
-    # The identifier is dependent on the compiled byte code of the
-    # function code (__code__.co_code), which can be different between
-    # python versions (normally, the content of the plugin file would be used).
-    if tuple(sys.version_info)[:3] < (3, 11, 0):
-        identifier = "3a3e72c4cb015424ebbe6d4af63f2170"
-    elif tuple(sys.version_info)[:3] < (3, 12, 0):
-        # bytecode changed in Python 3.11
-        identifier = "322e9a30665c8603ed2dd6fb9d7b18da"
-    else:
-        # bytecode changed in Python 3.12
-        identifier = "447e1ae998c716140f31614a4da2581f"
     # comparing lambda functions fails due to differing memory locations
     info.pop("method check required")
     pf = PlugInFeature("circ_per_area", info)
     pf.plugin_feature_info.pop("method check required")
+    # The identifier is dependent on the compiled byte code of the
+    # function code (__code__.co_code), which can be different between
+    # python versions and architectures.
+    identifier = pf.plugin_feature_info.pop("identifier")
+    assert len(identifier) == 32
     plugin_feature_info = {
         "method": compute_single_plugin_feature,
         "description": "This plugin will compute a feature",
@@ -162,8 +155,8 @@ def test_pf_attribute_plugin_feature_info():
         "scalar feature": True,
         "version": "0.1.0",
         "plugin path": None,
-        "identifier": identifier,
     }
+
     assert pf.plugin_feature_info == plugin_feature_info