dclab 0.62.9__tar.gz → 0.62.11__tar.gz

{dclab-0.62.9 → dclab-0.62.11}/CHANGELOG

@@ -1,3 +1,9 @@
+0.62.11
+ - enh: support plain `numpy.array` tables
+0.62.10
+ - fix: `basin_definition_copy` failed when more than one basins defined
+   due to redundant loop
+ - fix: JSON logs during export were not sorted and indented
 0.62.9
  - enh: sort basin definition before exporting to JSON; the side-effect
    is that the unique key for a basin changes, possibly resulting in

{dclab-0.62.9 → dclab-0.62.11}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: dclab
-Version: 0.62.9
+Version: 0.62.11
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dclab-0.62.9 → dclab-0.62.11}/dclab/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.62.9'
-__version_tuple__ = version_tuple = (0, 62, 9)
+__version__ = version = '0.62.11'
+__version_tuple__ = version_tuple = (0, 62, 11)

{dclab-0.62.9 → dclab-0.62.11}/dclab/rtdc_dataset/copier.py

@@ -178,42 +178,51 @@ def basin_definition_copy(src_h5file, dst_h5file, features_iter):
     relevant for the internal basin.
     """
     dst_h5file.require_group("basins")
-    for b_key in src_h5file["basins"]:
+    # Load the basin information
+    basin_dicts = RTDC_HDF5.basin_get_dicts_from_h5file(src_h5file)
+    for bn in basin_dicts:
+        b_key = bn["key"]
+
         if b_key in dst_h5file["basins"]:
-            # This basin already exists.
+            # already stored therein
             continue
-        # Load the basin information
-        basin_dicts = RTDC_HDF5.basin_get_dicts_from_h5file(src_h5file)
-        for bn in basin_dicts:
-            if bn["type"] == "internal":
-                # Make sure we define the internal features selected
-                feat_used = [f for f in bn["features"] if f in features_iter]
-                if len(feat_used) == 0:
-                    # We don't have any internal features, don't write anything
-                    continue
-                elif feat_used != bn["features"]:
-                    bn["features"] = feat_used
-                    rewrite = True
-                else:
-                    rewrite = False
+
+        # sanity check
+        if b_key not in src_h5file["basins"]:
+            raise ValueError(
+                f"Failed to parse basin information correctly. Source file "
+                f"{src_h5file} does not contain basin {b_key} which I got "
+                f"from `RTDC_HDF5.basin_get_dicts_from_h5file`.")
+
+        if bn["type"] == "internal":
+            # Make sure we define the internal features selected
+            feat_used = [f for f in bn["features"] if f in features_iter]
+            if len(feat_used) == 0:
+                # We don't have any internal features, don't write anything
+                continue
+            elif feat_used != bn["features"]:
+                bn["features"] = feat_used
+                rewrite = True
             else:
-                # We do not have an internal basin, just copy everything
                 rewrite = False
+        else:
+            # We do not have an internal basin, just copy everything
+            rewrite = False
 
-            if rewrite:
-                # Convert edited `bn` to JSON and write feature data
-                b_lines = json.dumps(bn, indent=2).split("\n")
-                key = hashobj(b_lines)
-                if key not in dst_h5file["basins"]:
-                    with RTDCWriter(dst_h5file) as hw:
-                        hw.write_text(dst_h5file["basins"], key, b_lines)
-            else:
-                # copy only
-                h5ds_copy(src_loc=src_h5file["basins"],
-                          src_name=b_key,
-                          dst_loc=dst_h5file["basins"],
-                          dst_name=b_key,
-                          recursive=False)
+        if rewrite:
+            # Convert edited `bn` to JSON and write feature data
+            b_lines = json.dumps(bn, indent=2).split("\n")
+            key = hashobj(b_lines)
+            if key not in dst_h5file["basins"]:
+                with RTDCWriter(dst_h5file) as hw:
+                    hw.write_text(dst_h5file["basins"], key, b_lines)
+        else:
+            # copy only
+            h5ds_copy(src_loc=src_h5file["basins"],
+                      src_name=b_key,
+                      dst_loc=dst_h5file["basins"],
+                      dst_name=b_key,
+                      recursive=False)
 
 
 def h5ds_copy(src_loc, src_name, dst_loc, dst_name=None,

{dclab-0.62.9 → dclab-0.62.11}/dclab/rtdc_dataset/export.py

@@ -319,18 +319,21 @@ class Export(object):
 
             # write export log
             hw.store_log(time.strftime("dclab-export_%Y-%m-%d_%H.%M.%S"),
-                         json.dumps({
-                             "dclab version": version_tuple,
-                             "kwargs": {
-                                 "features": features,
-                                 "filtered": filtered,
-                                 "logs": logs,
-                                 "tables": tables,
-                                 "basins": basins,
-                                 "meta_prefix": meta_prefix,
-                                 "skip_checks": skip_checks
-                             }
-                         }).split("\n"))
+                         json.dumps(
+                             {"dclab version": version_tuple,
+                              "kwargs": {
+                                  "features": features,
+                                  "filtered": filtered,
+                                  "logs": logs,
+                                  "tables": tables,
+                                  "basins": basins,
+                                  "meta_prefix": meta_prefix,
+                                  "skip_checks": skip_checks
+                                  }
+                              },
+                             indent=2,
+                             sort_keys=True,
+                         ).split("\n"))
 
             if logs:
                 # write logs

{dclab-0.62.9 → dclab-0.62.11}/dclab/rtdc_dataset/fmt_dcor/tables.py

@@ -30,11 +30,13 @@ class DCORTables:
             tables = {}
             for key in table_data:
                 columns, data = table_data[key]
-                ds_dt = np.dtype({'names': columns,
-                                  'formats': [np.float64] * len(columns)})
                 tab_data = np.asarray(data)
-                rec_arr = np.rec.array(tab_data, dtype=ds_dt)
-                tables[key] = rec_arr
+                if columns is not None:
+                    # We have a rec-array (named columns)
+                    ds_dt = np.dtype({'names': columns,
+                                      'formats': [np.float64] * len(columns)})
+                    tab_data = np.rec.array(tab_data, dtype=ds_dt)
+                tables[key] = tab_data
 
             self._tables_cache = tables
         return self._tables_cache

{dclab-0.62.9 → dclab-0.62.11}/dclab/rtdc_dataset/writer.py

@@ -616,7 +616,7 @@ class RTDCWriter:
                     convfunc = dfn.get_config_value_func(sec, ck)
                     self.h5file.attrs[idk] = convfunc(value)
 
-    def store_table(self, name, cmp_array):
+    def store_table(self, name, cmp_array, h5_attrs=None):
         """Store a compound array table
 
         Tables are semi-metadata. They may contain information collected
@@ -629,16 +629,33 @@ class RTDCWriter:
         ----------
         name: str
             Name of the table
-        cmp_array: np.recarray, h5py.Dataset, or dict
+        cmp_array: np.recarray, h5py.Dataset, np.ndarray, or dict
             If a np.recarray or h5py.Dataset are provided, then they
             are written as-is to the file. If a dictionary is provided,
             then the dictionary is converted into a numpy recarray.
+            If a numpy array is provided, then the array is written
+            as a raw table (no column names) to the file.
+        h5_attrs: dict, optional
+            Attributes to store alongside the corresponding HDF5 dataset
         """
-        if isinstance(cmp_array, (np.recarray, h5py.Dataset)):
+        if h5_attrs is None:
+            h5_attrs = {}
+
+        if isinstance(cmp_array, np.recarray):
             # A table is a compound array (np.recarray). If we are here,
-            # this means that the user passed an instance of np.recarray
-            # or an instance h5py.Dataset (which we trust to be a proper
+            # this means that the user passed an instance of np.recarray.
+            pass
+        elif isinstance(cmp_array, h5py.Dataset):
+            # An instance of h5py.Dataset (which we trust to be a proper
             # compound dataset at this point). No additional steps needed.
+            h5_attrs.update(cmp_array.attrs)
+            pass
+        elif isinstance(cmp_array, np.ndarray):
+            # A numpy array was passed. This usually means we have something
+            # that we can look at, so we add image tags.
+            h5_attrs['CLASS'] = np.bytes_('IMAGE')
+            h5_attrs['IMAGE_VERSION'] = np.bytes_('1.2')
+            h5_attrs['IMAGE_SUBCLASS'] = np.bytes_('IMAGE_GRAYSCALE')
             pass
         elif isinstance(cmp_array, dict):
             # The user passed a dict which we now have to convert to a
@@ -659,16 +676,18 @@ class RTDCWriter:
         else:
             raise NotImplementedError(
                 f"Cannot convert {type(cmp_array)} to table!")
+
+        # data
         group = self.h5file.require_group("tables")
         tab = group.create_dataset(
             name,
             data=cmp_array,
             fletcher32=True,
             **self.compression_kwargs)
-        # Also store metadata
-        if hasattr(cmp_array, "attrs"):
-            for key in cmp_array.attrs:
-                tab.attrs[key] = cmp_array.attrs[key]
+
+        # metadata
+        if h5_attrs:
+            tab.attrs.update(h5_attrs)
 
     def version_brand(self, old_version=None, write_attribute=True):
         """Perform version branding

{dclab-0.62.9 → dclab-0.62.11}/dclab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: dclab
-Version: 0.62.9
+Version: 0.62.11
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dclab-0.62.9 → dclab-0.62.11}/tests/test_cli_repack.py

@@ -80,6 +80,43 @@ def test_repack_basin_internal(use_basins):
             assert "userdef1" not in ds.features_basin
 
 
+@pytest.mark.filterwarnings(
+    "ignore::dclab.rtdc_dataset.config.WrongConfigurationTypeWarning")
+def test_repack_multiple_basins():
+    h5path = retrieve_data("fmt-hdf5_fl_wide-channel_2023.zip")
+    h5path_small = h5path.with_name("smaller.rtdc")
+    h5path_out = h5path.with_name("repacked.rtdc")
+
+    # Dataset creation
+    with h5py.File(h5path) as src, RTDCWriter(h5path_small) as hw:
+        # first, copy all the scalar features to the new file
+        rtdc_dataset.rtdc_copy(src_h5file=src,
+                               dst_h5file=hw.h5file,
+                               features="scalar")
+        hw.store_basin(basin_name="example basin",
+                       basin_type="file",
+                       basin_format="hdf5",
+                       basin_locs=[h5path],
+                       basin_descr="an example test basin",
+                       )
+
+        hw.store_basin(basin_name="example basin",
+                       basin_type="file",
+                       basin_format="hdf5",
+                       basin_locs=[h5path],
+                       basin_descr="a different test basin",
+                       )
+
+        assert len(hw.h5file["basins"].keys()) == 2
+
+    # repack the basin-based dataset
+    cli.repack(path_in=h5path_small,
+               path_out=h5path_out)
+
+    with new_dataset(h5path_out) as ds:
+        assert len(ds.basins) == 2
+
+
 def test_repack_no_data_from_basins_written():
     """
     When repacking a dataset, feature data from the basin should not be

{dclab-0.62.9 → dclab-0.62.11}/tests/test_rtdc_copier.py

@@ -203,6 +203,44 @@ def test_copy_basins_mapped():
         assert is_properly_compressed(hc["events/basinmap0"])
 
 
+def test_copy_basins_multiple():
+    path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
+    path_copy = path.with_name("test_copy.rtdc")
+
+    # add log to source file
+    with RTDCWriter(path, mode="append") as hw:
+        hw.store_basin(
+            basin_type="file",
+            basin_format="hdf5",
+            basin_name="test_basin",
+            basin_locs=["does-not-exist-but-does-not-matter.rtdc"],
+            basin_map=np.arange(len(hw.h5file["events/deform"])),
+            basin_descr="A first basin",
+            verify=False
+        )
+
+        hw.store_basin(
+            basin_type="file",
+            basin_format="hdf5",
+            basin_name="test_basin",
+            basin_locs=["does-not-exist-but-does-not-matter.rtdc"],
+            basin_map=np.arange(len(hw.h5file["events/deform"])),
+            basin_descr="A second basin",
+            verify=False
+        )
+
+        assert len(hw.h5file["basins"]) == 2
+
+    # copy
+    with h5py.File(path) as h5, h5py.File(path_copy, "w") as hc:
+        rtdc_copy(src_h5file=h5,
+                  dst_h5file=hc)
+
+    # Make sure this worked
+    with h5py.File(path_copy) as hc:
+        assert len(hc["basins"]) == 2
+
+
 def test_copy_basins_no_basin():
     path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
     path_copy = path.with_name("test_copy.rtdc")
@@ -419,6 +457,31 @@ def test_copy_tables():
             np.pi, 2 * np.pi, 10))
 
 
+def test_copy_tables_array_only():
+    path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
+    path_copy = path.with_name("test_copy.rtdc")
+
+    # generate a table that consists of an array, not a dict-like object
+    tab_data = np.random.random((1000, 300))
+
+    # add table to source file
+    with h5py.File(path, "a") as h5:
+        h5tab = h5.require_group("tables")
+        h5tab.create_dataset(name="random_data",
+                             data=tab_data)
+        assert not is_properly_compressed(h5["tables/random_data"])
+
+    # copy
+    with h5py.File(path) as h5, h5py.File(path_copy, "w") as hc:
+        rtdc_copy(src_h5file=h5,
+                  dst_h5file=hc)
+
+    # Make sure this worked
+    with h5py.File(path_copy) as hc:
+        assert is_properly_compressed(hc["tables/random_data"])
+        assert np.all(tab_data == hc["tables/random_data"])
+
+
 def test_copy_tables_hdf5_issue_3214():
     """Checks for a bug in HDF5
 

{dclab-0.62.9 → dclab-0.62.11}/tests/test_rtdc_export_hdf5.py

@@ -609,6 +609,27 @@ def test_hdf5_tables(tables):
             assert "src_iratrax" not in h5.get("tables", {})
 
 
+def test_hdf5_tables_array_only():
+    keys = ["area_um", "deform", "time", "frame", "index_online"]
+    ddict = example_data_dict(size=10, keys=keys)
+    ds1 = dclab.new_dataset(ddict)
+    ds1.config["experiment"]["sample"] = "test"
+    ds1.config["experiment"]["run index"] = 1
+    ds1.config["imaging"]["frame rate"] = 2000
+
+    # generate a table that consists of an array, not a dict-like object
+    tab_data = np.random.random((1000, 300))
+    ds1.tables["iratrax"] = tab_data
+
+    edest = pathlib.Path(tempfile.mkdtemp())
+    f1 = edest / "dclab_test_export_hdf5_1.rtdc"
+    ds1.export.hdf5(f1, keys, tables=True)
+    with h5py.File(f1, "r") as h5:
+        assert "src_iratrax" in h5.get("tables", {})
+        assert np.all(h5["tables"]["src_iratrax"] == tab_data)
+        assert h5["tables"]["src_iratrax"].attrs["CLASS"] == np.bytes_("IMAGE")
+
+
 def test_hdf5_trace_from_tdms():
     pytest.importorskip("nptdms")
     ds = new_dataset(retrieve_data("fmt-tdms_2fl-no-image_2017.zip"))

{dclab-0.62.9 → dclab-0.62.11}/tests/test_rtdc_feat_basin.py

@@ -410,7 +410,12 @@ def test_basin_key_reproducible():
     with dclab.new_dataset(h5path_small_2) as ds:
         assert len(ds.basins) == 1
         # dclab 0.62.9
-        assert ds.basins[0].key == "b8763fad05e17b58407875fe9c28d90b"
+        assert ds.basins[0].key in [
+            # posix linesep
+            "b8763fad05e17b58407875fe9c28d90b",
+            # win linesep
+            "a495fd2d495e0d991f3a4a313ec3a3fc",
+        ]
 
 
 def test_basin_sorting_basic():