dclab 0.62.17__tar.gz → 0.63.0__tar.gz

{dclab-0.62.17 → dclab-0.63.0}/CHANGELOG

@@ -1,3 +1,5 @@
+0.63.0
+ - feat: add `progress_callback` argument to all data export methods (#271)
 0.62.17
  - docs: properly employ license GPL-2.0-or-later
  - ref: add KernelDensityEstimator class

{dclab-0.62.17 → dclab-0.63.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dclab
-Version: 0.62.17
+Version: 0.63.0
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dclab-0.62.17 → dclab-0.63.0}/dclab/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.62.17'
-__version_tuple__ = version_tuple = (0, 62, 17)
+__version__ = version = '0.63.0'
+__version_tuple__ = version_tuple = (0, 63, 0)

{dclab-0.62.17 → dclab-0.63.0}/dclab/rtdc_dataset/export.py

@@ -51,6 +51,7 @@ class Export(object):
             pixel_format: str = "yuv420p",
             codec: str = "rawvideo",
             codec_options: dict[str, str] = None,
+            progress_callback: callable = None,
             ):
         """Exports filtered event images to a video file
 
@@ -72,6 +73,10 @@ class Export(object):
         codec_options:
             Additional arguments to give to the codec using ffmpeg,
             e.g. `{'preset': 'slow', 'crf': '0'}` for "libx264" codec.
+        progress_callback: callable
+            Function that takes at least two arguments: float between 0 and
+            1 for monitoring progress and a string describing what is being
+            done.
 
         Notes
         -----
@@ -103,6 +108,10 @@ class Export(object):
 
                 # write the filtered frames to the video file
                 for evid in np.arange(len(ds)):
+
+                    if progress_callback is not None and evid % 10_000 == 0:
+                        progress_callback(evid / len(ds), "exporting video")
+
                     # skip frames that were filtered out
                     if filtered and not ds.filter.all[evid]:
                         continue
@@ -116,12 +125,22 @@ class Export(object):
 
                     for packet in stream.encode(av_frame):
                         container.mux(packet)
+
+            if progress_callback is not None:
+                progress_callback(1.0, "video export complete")
+
         else:
             msg = "No image data to export: dataset {} !".format(ds.title)
             raise OSError(msg)
 
-    def fcs(self, path, features, meta_data=None, filtered=True,
-            override=False):
+    def fcs(self,
+            path: pathlib.Path | str,
+            features: list[str],
+            meta_data: dict = None,
+            filtered: bool = True,
+            override: bool = False,
+            progress_callback: callable = None,
+            ):
         """Export the data of an RT-DC dataset to an .fcs file
 
         Parameters
@@ -142,6 +161,10 @@ class Export(object):
         override: bool
             If set to `True`, an existing file ``path`` will be overridden.
             If set to `False`, raises `OSError` if ``path`` exists.
+        progress_callback: callable
+            Function that takes at least two arguments: float between 0 and
+            1 for monitoring progress and a string describing what is being
+            done.
 
         Notes
         -----
@@ -175,12 +198,18 @@ class Export(object):
         # Collect the header
         chn_names = [dfn.get_feature_label(c, rtdc_ds=ds) for c in features]
 
+        if progress_callback is not None:
+            progress_callback(0.0, "collecting data")
+
         # Collect the data
         if filtered:
             data = [ds[c][ds.filter.all] for c in features]
         else:
             data = [ds[c] for c in features]
 
+        if progress_callback is not None:
+            progress_callback(0.5, "exporting data")
+
         data = np.array(data).transpose()
         meta_data["dclab version"] = version
         fcswrite.write_fcs(filename=str(path),
@@ -189,6 +218,9 @@ class Export(object):
                            text_kw_pr=meta_data,
                            )
 
+        if progress_callback is not None:
+            progress_callback(1.0, "export complete")
+
     def hdf5(self,
              path: str | pathlib.Path,
              features: List[str] = None,
@@ -200,7 +232,9 @@ class Export(object):
              override: bool = False,
              compression_kwargs: Dict = None,
              compression: str = "deprecated",
-             skip_checks: bool = False):
+             skip_checks: bool = False,
+             progress_callback: callable = None,
+             ):
         """Export the data of the current instance to an HDF5 file
 
         Parameters
@@ -244,7 +278,10 @@ class Export(object):
                 Use `compression_kwargs` instead.
         skip_checks: bool
             Disable checking whether all features have the same length.
-
+        progress_callback: callable
+            Function that takes at least two arguments: float between 0 and
+            1 for monitoring progress and a string describing what is being
+            done.
 
         .. versionchanged:: 0.58.0
 
@@ -335,6 +372,8 @@ class Export(object):
         with RTDCWriter(path,
                         mode="append",
                         compression_kwargs=compression_kwargs) as hw:
+            if progress_callback is not None:
+                progress_callback(0.0, "writing metadata")
             # write meta data
             hw.store_metadata(meta)
 
@@ -369,7 +408,10 @@ class Export(object):
                                    ds.tables[tab])
 
             # write each feature individually
-            for feat in features:
+            for ii, feat in enumerate(features):
+                if progress_callback is not None:
+                    progress_callback(ii / len(features), f"exporting {feat}")
+
                 if (filter_arr is None or
                         # This does not work for the .tdms file format
                         # (and probably also not for DCOR).
@@ -393,6 +435,10 @@ class Export(object):
                                            filtarr=filter_arr)
 
             if basins:
+                if progress_callback:
+                    progress_callback(1 - 1 / (len(features) or 1),
+                                      "writing basins")
+
                 # We have to store basins. There are three options:
                 # - filtering disabled: just copy basins
                 # - filtering enabled
@@ -472,9 +518,17 @@ class Export(object):
 
                     # Do not verify basins, it takes too long.
                     hw.store_basin(**bn_dict, verify=False)
+        if progress_callback is not None:
+            progress_callback(1.0, "export complete")
 
-    def tsv(self, path, features, meta_data=None, filtered=True,
-            override=False):
+    def tsv(self,
+            path: pathlib.Path | str,
+            features: list[str],
+            meta_data: dict = None,
+            filtered: bool = True,
+            override: bool = False,
+            progress_callback: callable = None,
+            ):
         """Export the data of the current instance to a .tsv file
 
         Parameters
@@ -496,6 +550,10 @@ class Export(object):
         override: bool
             If set to `True`, an existing file ``path`` will be overridden.
             If set to `False`, raises `OSError` if ``path`` exists.
+        progress_callback: callable
+            Function that takes at least two arguments: float between 0 and
+            1 for monitoring progress and a string describing what is being
+            done.
         """
         if meta_data is None:
             meta_data = {}
@@ -516,6 +574,10 @@ class Export(object):
             if c not in ds.features_scalar:
                 raise ValueError("Invalid feature name {}".format(c))
         meta_data["dclab version"] = version
+
+        if progress_callback is not None:
+            progress_callback(0.0, "writing metadata")
+
         # Write BOM header
         with path.open("wb") as fd:
             fd.write(codecs.BOM_UTF8)
@@ -539,17 +601,26 @@ class Export(object):
             fd.write("# "+header2+"\n")
 
         with path.open("ab") as fd:
-            # write data
+            if progress_callback is not None:
+                progress_callback(0.1, "collecting data")
+
+            # collect data
             if filtered:
                 data = [ds[c][ds.filter.all] for c in features]
             else:
                 data = [ds[c] for c in features]
 
+            if progress_callback is not None:
+                progress_callback(0.5, "writing data")
+
             np.savetxt(fd,
                        np.array(data).transpose(),
                        fmt=str("%.10e"),
                        delimiter="\t")
 
+        if progress_callback is not None:
+            progress_callback(1.0, "export complete")
+
 
 def yield_filtered_array_stacks(data, indices):
     """Generator returning chunks with the filtered feature data

{dclab-0.62.17 → dclab-0.63.0}/dclab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dclab
-Version: 0.62.17
+Version: 0.63.0
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dclab-0.62.17 → dclab-0.63.0}/tests/test_rtdc_export_avi.py

@@ -1,11 +1,9 @@
-import os
-from os.path import join
-import tempfile
+from unittest import mock
 
 import pytest
 
 import dclab
-from dclab import new_dataset
+from dclab import new_dataset, RTDCWriter
 import numpy as np
 
 from helper_methods import example_data_dict, retrieve_data
@@ -30,48 +28,72 @@ def test_avi_export_check(tmp_path):
     assert num_frames == len(ds)
 
 
+@pytest.mark.filterwarnings(
+    "ignore::dclab.rtdc_dataset.config.WrongConfigurationTypeWarning")
+def test_avi_export_progress(tmp_path):
+    path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
+    # create an .rtdc file with more than 10_000 events
+    path_large = path.with_name("large.rtdc")
+    path_avi = path.with_name("export.avi")
+
+    with RTDCWriter(path_large) as hw, new_dataset(path) as ds:
+        hw.store_metadata(ds.config.as_dict(pop_filtering=True))
+        size = len(ds)
+        num_iters = 25_000 // size
+        for feat in ["time", "image", "deform"]:
+            feat_data = np.concatenate([ds[feat][:]] * num_iters)
+            hw.store_feature(feat, feat_data)
+
+    with dclab.new_dataset(path_large) as dse:
+        assert len(dse) > 20_000, "we need a large dataset"
+
+        callback = mock.MagicMock()
+        dse.export.avi(path=path_avi, progress_callback=callback)
+
+    callback.assert_any_call(0.0, "exporting video")
+    callback.assert_any_call(0.4, "exporting video")
+    callback.assert_any_call(0.8, "exporting video")
+    callback.assert_any_call(1.0, "video export complete")
+
+
 @pytest.mark.filterwarnings(
     "ignore::dclab.rtdc_dataset.config.WrongConfigurationTypeWarning")
 @pytest.mark.filterwarnings('ignore::dclab.rtdc_dataset.'
                             + 'fmt_tdms.exc.CorruptFrameWarning')
-def test_avi_export_tdms():
+def test_avi_export_tdms(tmp_path):
     pytest.importorskip("nptdms")
     ds = new_dataset(retrieve_data("fmt-tdms_fl-image_2016.zip"))
-    edest = tempfile.mkdtemp()
-    f1 = join(edest, "test.avi")
+
+    f1 = tmp_path / "test.avi"
     ds.export.avi(path=f1)
-    assert os.stat(
-        f1)[6] > 1e4, "Resulting file to small, Something went wrong!"
+    assert f1.stat()[6] > 1e4, "Resulting file to small, Something went wrong!"
 
 
 @pytest.mark.filterwarnings(
     "ignore::dclab.rtdc_dataset.config.WrongConfigurationTypeWarning")
 @pytest.mark.filterwarnings('ignore::dclab.rtdc_dataset.'
                             + 'fmt_tdms.exc.CorruptFrameWarning')
-def test_avi_override_tdms():
+def test_avi_override_tdms(tmp_path):
     pytest.importorskip("nptdms")
     ds = new_dataset(retrieve_data("fmt-tdms_fl-image_2016.zip"))
 
-    edest = tempfile.mkdtemp()
-    f1 = join(edest, "test.avi")
+    f1 = tmp_path / "test.avi"
     ds.export.avi(f1, override=True)
     try:
-        ds.export.avi(f1[:-4], override=False)
+        ds.export.avi(f1.with_name(f1.stem), override=False)
     except OSError:
         pass
     else:
         raise ValueError("Should append .avi and not override!")
 
 
-def test_avi_no_images():
+def test_avi_no_images(tmp_path):
     keys = ["area_um", "deform", "time", "frame", "fl3_width"]
     ddict = example_data_dict(size=127, keys=keys)
     ds = dclab.new_dataset(ddict)
 
-    edest = tempfile.mkdtemp()
-    f1 = join(edest, "test.avi")
     try:
-        ds.export.avi(f1)
+        ds.export.avi(tmp_path / "test.avi")
     except OSError:
         pass
     else:

{dclab-0.62.17 → dclab-0.63.0}/tests/test_rtdc_export_fcs.py

@@ -1,6 +1,7 @@
 import io
 from os.path import join
 import tempfile
+from unittest import mock
 
 import pytest
 
@@ -33,6 +34,22 @@ def test_fcs_export():
     assert len(a1) != 0
 
 
+def test_fcs_export_progress(tmp_path):
+    pytest.importorskip("fcswrite")
+    keys = ["area_um", "deform", "time", "frame", "fl3_width"]
+    ddict = example_data_dict(size=222, keys=keys)
+    ds = dclab.new_dataset(ddict)
+
+    f1 = tmp_path / "test.fcs"
+
+    callback = mock.MagicMock()
+    ds.export.fcs(f1, keys, progress_callback=callback)
+
+    callback.assert_any_call(0.0, "collecting data")
+    callback.assert_any_call(0.5, "exporting data")
+    callback.assert_any_call(1.0, "export complete")
+
+
 def test_fcs_override():
     pytest.importorskip("fcswrite")
     keys = ["area_um", "deform", "time", "frame", "fl3_width"]

{dclab-0.62.17 → dclab-0.63.0}/tests/test_rtdc_export_hdf5.py

@@ -2,6 +2,7 @@ import json
 from os.path import join
 import pathlib
 import tempfile
+from unittest import mock
 
 import h5py
 import numpy as np
@@ -578,6 +579,63 @@ def test_hdf5_override():
         raise ValueError("Should append .rtdc and not override!")
 
 
+def test_hdf5_progress(tmp_path):
+    keys = ["area_um", "deform", "time", "frame", "fl3_width"]
+    ddict = example_data_dict(size=222, keys=keys)
+    ds = dclab.new_dataset(ddict)
+
+    f1 = tmp_path / "test.rtdc"
+
+    callback = mock.MagicMock()
+    ds.export.hdf5(f1, keys, progress_callback=callback)
+
+    callback.assert_any_call(0.0, "writing metadata")
+    callback.assert_any_call(0/5, "exporting area_um")
+    callback.assert_any_call(1/5, "exporting deform")
+    callback.assert_any_call(2/5, "exporting fl3_width")
+    callback.assert_any_call(3/5, "exporting frame")
+    callback.assert_any_call(4/5, "exporting time")
+    callback.assert_any_call(1.0, "export complete")
+
+
+def test_hdf5_progress_basins(tmp_path):
+    h5path = retrieve_data("fmt-hdf5_image-mask-blood_2021.zip")
+    path_exp = h5path.with_name("exported.rtdc")
+
+    callback = mock.MagicMock()
+
+    with dclab.new_dataset(h5path) as ds:
+        ds.export.hdf5(path_exp,
+                       features=["deform", "image"],
+                       basins=True,
+                       progress_callback=callback
+                       )
+
+    callback.assert_any_call(0.0, "writing metadata")
+    callback.assert_any_call(0/2, "exporting deform")
+    callback.assert_any_call(1/2, "exporting image")
+    callback.assert_any_call(1/2, "writing basins")
+    callback.assert_any_call(1.0, "export complete")
+
+
+def test_hdf5_progress_basins_only(tmp_path):
+    h5path = retrieve_data("fmt-hdf5_image-mask-blood_2021.zip")
+    path_exp = h5path.with_name("exported.rtdc")
+
+    callback = mock.MagicMock()
+
+    with dclab.new_dataset(h5path) as ds:
+        ds.export.hdf5(path_exp,
+                       features=[],
+                       basins=True,
+                       progress_callback=callback
+                       )
+
+    callback.assert_any_call(0.0, "writing metadata")
+    callback.assert_any_call(0.0, "writing basins")
+    callback.assert_any_call(1.0, "export complete")
+
+
 @pytest.mark.parametrize("tables", [True, False])
 def test_hdf5_tables(tables):
     keys = ["area_um", "deform", "time", "frame", "index_online"]

{dclab-0.62.17 → dclab-0.63.0}/tests/test_rtdc_export_tsv.py

@@ -1,3 +1,4 @@
+from unittest import mock
 import dclab
 
 from helper_methods import example_data_dict, retrieve_data
@@ -27,6 +28,23 @@ def test_tsv_export(tmp_path):
     assert len(a1) != 0
 
 
+def test_fcs_export_progress(tmp_path):
+    pytest.importorskip("fcswrite")
+    keys = ["area_um", "deform", "time", "frame", "fl3_width"]
+    ddict = example_data_dict(size=222, keys=keys)
+    ds = dclab.new_dataset(ddict)
+
+    f1 = tmp_path / "test.tsv"
+
+    callback = mock.MagicMock()
+    ds.export.tsv(f1, keys, progress_callback=callback)
+
+    callback.assert_any_call(0.0, "writing metadata")
+    callback.assert_any_call(0.1, "collecting data")
+    callback.assert_any_call(0.5, "writing data")
+    callback.assert_any_call(1.0, "export complete")
+
+
 @pytest.mark.filterwarnings(
     "ignore::dclab.rtdc_dataset.config.WrongConfigurationTypeWarning")
 def test_tsv_export_ds_metadata(tmp_path):