dcnum 0.23.1__py3-none-any.whl → 0.25.1__py3-none-any.whl

dcnum/write/writer.py

@@ -1,13 +1,14 @@
 import hashlib
 import json
 import pathlib
-from typing import Dict, List
+from typing import Dict, List, Tuple
 import warnings
 
 import h5py
 import hdf5plugin
 import numpy as np
 
+from ..read import HDF5Data
 from .._version import version
 
 
@@ -16,6 +17,11 @@ class CreatingFileWithoutBasinWarning(UserWarning):
     pass
 
 
+class IgnoringBasinTypeWarning(UserWarning):
+    """Issued when a specific basin type is ignored"""
+    pass
+
+
 class HDF5Writer:
     def __init__(self,
                  # TODO: make this a mandatory argument when `path` is
@@ -48,7 +54,13 @@ class HDF5Writer:
             self.h5 = obj
             self.h5_owned = False
         else:
-            self.h5 = h5py.File(obj, mode=mode, libver="latest")
+            self.h5 = h5py.File(obj,
+                                mode=mode,
+                                libver="latest",
+                                # Set chunk cache size to 3 MiB for each
+                                # dataset to allow partial writes.
+                                rdcc_nbytes=3145728,
+                                )
             self.h5_owned = True
         self.events = self.h5.require_group("events")
         ds_kwds = set_default_filter_kwargs(ds_kwds)
@@ -84,14 +96,42 @@ class HDF5Writer:
         chunk_size_int = max(10, int(np.floor(chunk_size)))
         return tuple([chunk_size_int] + list(item_shape))
 
-    def require_feature(self, feat, item_shape, feat_dtype, ds_kwds=None):
-        """Create a new feature in the "events" group"""
-        if feat not in self.events:
+    def require_feature(self,
+                        feat: str,
+                        item_shape: Tuple[int],
+                        feat_dtype: np.dtype,
+                        ds_kwds: Dict = None,
+                        group_name: str = "events"):
+        """Create a new feature in the "events" group
+
+        Parameters
+        ----------
+        feat: str
+            name of the feature
+        item_shape: Tuple[int]
+            shape for one event of this feature, e.g. for a scalar
+            event, the shape would be `(1,)` and for an image, the
+            shape could be `(80, 300)`.
+        feat_dtype: np.dtype
+            dtype of the feature
+        ds_kwds: Dict
+            HDF5 Dataset keyword arguments (e.g. compression, fletcher32)
+        group_name: str
+            name of the HDF5 group where the feature should be written to;
+            defaults to the "events" group, but a different group can be
+            specified for storing e.g. internal  basin features.
+        """
+        if group_name == "events":
+            egroup = self.events
+        else:
+            egroup = self.h5.require_group(group_name)
+
+        if feat not in egroup:
             if ds_kwds is None:
                 ds_kwds = {}
             for key in self.ds_kwds:
                 ds_kwds.setdefault(key, self.ds_kwds[key])
-            dset = self.events.create_dataset(
+            dset = egroup.create_dataset(
                 feat,
                 shape=tuple([0] + list(item_shape)),
                 dtype=feat_dtype,
@@ -100,22 +140,23 @@ class HDF5Writer:
                                                feat_dtype=feat_dtype),
                 **ds_kwds)
             if len(item_shape) == 2:
-                dset.attrs.create('CLASS', np.string_('IMAGE'))
-                dset.attrs.create('IMAGE_VERSION', np.string_('1.2'))
+                dset.attrs.create('CLASS', np.bytes_('IMAGE'))
+                dset.attrs.create('IMAGE_VERSION', np.bytes_('1.2'))
                 dset.attrs.create('IMAGE_SUBCLASS',
-                                  np.string_('IMAGE_GRAYSCALE'))
+                                  np.bytes_('IMAGE_GRAYSCALE'))
             offset = 0
         else:
-            dset = self.events[feat]
+            dset = egroup[feat]
             offset = dset.shape[0]
         return dset, offset
 
     def store_basin(self,
                     name: str,
-                    paths: List[str | pathlib.Path],
+                    paths: List[str | pathlib.Path] | None = None,
                     features: List[str] = None,
                     description: str | None = None,
-                    mapping: np.ndarray = None
+                    mapping: np.ndarray = None,
+                    internal_data: Dict | None = None,
                     ):
         """Write an HDF5-based file basin
 
@@ -123,8 +164,9 @@ class HDF5Writer:
         ----------
         name: str
             basin name; Names do not have to be unique.
-        paths: list of str or pathlib.Path
-            location(s) of the basin
+        paths: list of str or pathlib.Path or None
+            location(s) of the basin; must be None when storing internal
+            data, a list of paths otherwise
         features: list of str
             list of features provided by `paths`
         description: str
@@ -132,14 +174,39 @@ class HDF5Writer:
         mapping: 1D array
             integer array with indices that map the basin dataset
             to this dataset
+        internal_data: dict of ndarrays
+            internal basin data to store; If this is set, then `features`
+            and `paths` must be set to `None`.
         """
         bdat = {
             "description": description,
-            "format": "hdf5",
             "name": name,
-            "paths": [str(pp) for pp in paths],
-            "type": "file",
         }
+
+        if internal_data:
+            if features is not None:
+                raise ValueError("`features` must be set to None when storing "
+                                 "internal basin features")
+            if paths is not None:
+                raise ValueError("`paths` must be set to None when storing "
+                                 "internal basin features")
+            # store the internal basin information
+            for feat in internal_data:
+                if feat in self.h5.require_group("basin_events"):
+                    raise ValueError(f"Feature '{feat}' is already defined "
+                                     f"as an internal basin feature")
+                self.store_feature_chunk(feat=feat,
+                                         data=internal_data[feat],
+                                         group_name="basin_events")
+            features = sorted(internal_data.keys())
+            bdat["format"] = "h5dataset"
+            bdat["paths"] = ["basin_events"]
+            bdat["type"] = "internal"
+        else:
+            bdat["format"] = "hdf5"
+            bdat["paths"] = [str(pp) for pp in paths]
+            bdat["type"] = "file"
+
         # Explicit features stored in basin file
         if features is not None and len(features):
             bdat["features"] = features
@@ -189,7 +256,7 @@ class HDF5Writer:
                 chunks=True,
                 **self.ds_kwds)
 
-    def store_feature_chunk(self, feat, data):
+    def store_feature_chunk(self, feat, data, group_name="events"):
         """Store feature data
 
         The "chunk" implies that always chunks of data are stored,
@@ -199,7 +266,8 @@ class HDF5Writer:
             data = 255 * np.array(data, dtype=np.uint8)
         ds, offset = self.require_feature(feat=feat,
                                           item_shape=data.shape[1:],
-                                          feat_dtype=data.dtype)
+                                          feat_dtype=data.dtype,
+                                          group_name=group_name)
         dsize = data.shape[0]
         ds.resize(offset + dsize, axis=0)
         ds[offset:offset + dsize] = data
@@ -285,14 +353,17 @@ def create_with_basins(
                 # copy metadata
                 with h5py.File(prep, libver="latest") as h5:
                     copy_metadata(h5_src=h5, h5_dst=hw.h5)
+                    copy_basins(h5_src=h5, h5_dst=hw.h5)
                     # extract features
                     features = sorted(h5["events"].keys())
+                    features = [f for f in features if
+                                not f.startswith("basinmap")]
                 name = prep.name
             else:
                 features = None
                 name = bps[0]
 
-            # Finally, write the basin.
+            # Write the basin data
             hw.store_basin(name=name,
                            paths=bps,
                            features=features,
@@ -300,6 +371,44 @@ def create_with_basins(
                            )
 
 
+def copy_basins(h5_src: h5py.File,
+                h5_dst: h5py.File,
+                internal_basins: bool = True
+                ):
+    """Reassemble basin data in the output file
+
+    This does not just copy the datasets defined in the "basins"
+    group, but it also loads the "basinmap?" features and stores
+    them as new "basinmap?" features in the output file.
+    """
+    basins = HDF5Data.extract_basin_dicts(h5_src, check=False)
+    hw = HDF5Writer(h5_dst)
+    for bn_dict in basins:
+        if bn_dict["type"] == "internal" and internal_basins:
+            internal_data = {}
+            for feat in bn_dict["features"]:
+                internal_data[feat] = h5_src["basin_events"][feat]
+            hw.store_basin(name=bn_dict["name"],
+                           description=bn_dict["description"],
+                           mapping=h5_src["events"][bn_dict["mapping"]][:],
+                           internal_data=internal_data,
+                           )
+        elif bn_dict["type"] == "file":
+            if bn_dict.get("mapping") is not None:
+                mapping = h5_src["events"][bn_dict["mapping"]][:]
+            else:
+                mapping = None
+            hw.store_basin(name=bn_dict["name"],
+                           description=bn_dict["description"],
+                           paths=bn_dict["paths"],
+                           features=bn_dict["features"],
+                           mapping=mapping,
+                           )
+        else:
+            warnings.warn(f"Ignored basin of type '{bn_dict['type']}'",
+                          IgnoringBasinTypeWarning)
+
+
 def copy_features(h5_src: h5py.File,
                   h5_dst: h5py.File,
                   features: List[str],
@@ -323,8 +432,6 @@ def copy_features(h5_src: h5py.File,
     """
     ei = h5_src["events"]
     eo = h5_dst.require_group("events")
-    # This is the size of the output dataset
-    size = h5_dst.attrs["experiment:event count"]
     hw = HDF5Writer(h5_dst)
     for feat in features:
         if feat in eo:
@@ -341,26 +448,34 @@ def copy_features(h5_src: h5py.File,
                           dst_name=feat.encode(),
                           )
         else:
-            # Perform mapping and store the features in chunks to keep
-            # memory usage down.
+            # We have to perform mapping.
+            # Since h5py is very slow at indexing with arrays,
+            # we instead read the data in chunks from the input file,
+            # and perform the mapping afterward using the numpy arrays.
             dsi = ei[feat]
             chunk_size = hw.get_best_nd_chunks(dsi[0].shape, dsi.dtype)[0]
+            size_in = dsi.shape[0]
             start = 0
-            while start < size:
-                chunk_idx = mapping[start:start + chunk_size]
-                # h5py only supports indexing in increasing order
-                chunk_unique, order = np.unique(chunk_idx, return_inverse=True)
-                data_unique = dsi[chunk_unique]
-                data = data_unique[order]
+            while start < size_in:
+                # Get a big chunk of data
+                big_chunk = 10 * chunk_size
+                stop = start + big_chunk
+                data_in = dsi[start:stop]
+                # Determine the indices that we need from that chunk.
+                mapping_idx = (start <= mapping) * (mapping < stop)
+                mapping_chunk = mapping[mapping_idx] - start
+                data = data_in[mapping_chunk]
+                # Note that HDF5 does its own caching, properly handling
+                # partial chunk writes.
                 hw.store_feature_chunk(feat, data)
                 # increment start
-                start += chunk_size
+                start = stop
 
 
 def copy_metadata(h5_src: h5py.File,
-                  h5_dst: h5py.File,
-                  copy_basins=True):
-    """Copy attributes, tables, logs, and basins from one H5File to another
+                  h5_dst: h5py.File
+                  ):
+    """Copy attributes, tables, and logs from one H5File to another
 
     Notes
     -----
@@ -374,8 +489,6 @@ def copy_metadata(h5_src: h5py.File,
     for kk in src_attrs:
         h5_dst.attrs.setdefault(kk, src_attrs[kk])
     copy_data = ["logs", "tables"]
-    if copy_basins:
-        copy_data.append("basins")
     # copy other metadata
     for topic in copy_data:
         if topic in h5_src:

{dcnum-0.23.1.dist-info → dcnum-0.25.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.23.1
+Version: 0.25.1
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -26,7 +26,7 @@ Requires-Dist: opencv-python-headless
 Requires-Dist: scikit-image
 Requires-Dist: scipy >=1.8.0
 Provides-Extra: torch
-Requires-Dist: torch >=2.3 ; extra == 'torch'
+Requires-Dist: torch >=2.2 ; extra == 'torch'
 
 |dcnum|
 =======

{dcnum-0.23.1.dist-info → dcnum-0.25.1.dist-info}/RECORD

@@ -1,14 +1,14 @@
 dcnum/__init__.py,sha256=hcawIKS7utYiOyVhOAX9t7K3xYzP1b9862VV0b6qSrQ,74
-dcnum/_version.py,sha256=FvSfCBPqaPzqv_nSHIaBJNolXopVkF1cRxifqhCXk4g,413
+dcnum/_version.py,sha256=uQMGTxwBAv62qqmqUtTzCRwELoTJrxD49IfmR-dUSn8,413
 dcnum/feat/__init__.py,sha256=jUJYWTD3VIoDNKrmryXbjHb1rGwYtK4b7VPWihYgUoo,325
-dcnum/feat/event_extractor_manager_thread.py,sha256=5HdCQCywyQ5QC56AMjSqCroqif9oOFyiSFWCe07JojM,7820
+dcnum/feat/event_extractor_manager_thread.py,sha256=FAxSyRfaNAuBWNplxHngp5h-44s0qIP24XX_oETdfMk,7836
 dcnum/feat/gate.py,sha256=Yhxq80JoRMmQzBxl35C8NT91c9QcmQa-EIKLuxK6WvE,7221
 dcnum/feat/queue_event_extractor.py,sha256=0ncTQleT1sfc98zYkFuZWxU-akecfTrW6-OOU3z-d8o,15698
 dcnum/feat/feat_background/__init__.py,sha256=OTmMuazHNaSrZb2XW4cnJ6PlgJLbKrPbaidpEixYa0A,341
-dcnum/feat/feat_background/base.py,sha256=A-K3qlJ0ABFBGm5eMKYcNCC7ktFAInSm0eR3N3DHQZY,7963
+dcnum/feat/feat_background/base.py,sha256=bQBPvztrku-8YSVk8YBUUNh7MaYcnztgyD2-dQHxpzw,8674
 dcnum/feat/feat_background/bg_copy.py,sha256=PK8x4_Uph-_A6uszZC5uhe1gD1dSRdHnDMEsN0HSGHA,1034
 dcnum/feat/feat_background/bg_roll_median.py,sha256=EyjstMDXFBYuJB1lN6g4Uw7tPm434X3hXQxKSqvcoJ4,13175
-dcnum/feat/feat_background/bg_sparse_median.py,sha256=ab7Boj7cmr6PBdTbyWTj_yNNJSfuowr7u-iSGW989WI,20709
+dcnum/feat/feat_background/bg_sparse_median.py,sha256=wt7lvPSiZkdaerRErgezd_YDVxHA2kAqO1LMX9PuHJk,22053
 dcnum/feat/feat_brightness/__init__.py,sha256=o6AebVlmydwNgVF5kW6ITqJyFreoKrU3Ki_3EC8If-s,155
 dcnum/feat/feat_brightness/bright_all.py,sha256=vf8xaYBdKD24hHUXdkI0_S7nbr7m49KW6gvuWvbHDVg,4545
 dcnum/feat/feat_brightness/common.py,sha256=JX49EszYDmnvoOKXFVV1CalEIWRmOuY5EryNbqGbdac,156
@@ -20,36 +20,37 @@ dcnum/feat/feat_texture/__init__.py,sha256=6StM9S540UVtdFFR3bHa7nfCTomeVdoo7Uy9C
 dcnum/feat/feat_texture/common.py,sha256=COXHpXS-7DMouGu3WF83I76L02Sr7P9re4lxajh6g0E,439
 dcnum/feat/feat_texture/tex_all.py,sha256=_5H3sXYRN0Uq2eUHn3XUyEHkU_tncEqbqJTC-HZcnGY,5198
 dcnum/logic/__init__.py,sha256=7J3GrwJInNQbrLk61HRIV7X7p69TAIbMYpR34hh6u14,177
-dcnum/logic/ctrl.py,sha256=FyVlizHOIaIGMqINvM-outPywAQU0-5NM7t1dEDml4c,35332
-dcnum/logic/job.py,sha256=H1uDZ1nnNHNWWCe6mS8OWB0Uxc6XUKLISx5xExeplZY,7015
+dcnum/logic/ctrl.py,sha256=FwzrCou4YBPBaaRRTqJkPt6jenFev9JfnxxjUMS0B-w,39042
+dcnum/logic/job.py,sha256=9BN2WjYqjjJuLnfNZAtQ2Nn47Glo2jVrivDodGJoqlQ,7713
 dcnum/logic/json_encoder.py,sha256=cxMnqisbKEVf-rVcw6rK2BBAb6iz_hKFaGl81kK36lQ,571
 dcnum/meta/__init__.py,sha256=AVqRgyKXO1orKnE305h88IBvoZ1oz6X11HN1WP5nGvg,60
 dcnum/meta/paths.py,sha256=J_ikeHzd7gEeRgAKjuayz3x6q4h1fOiDadM-ZxhAGm4,1053
-dcnum/meta/ppid.py,sha256=OD79NrZ8waC3julwdH8NjneUuXqSRSHsUDpKzT5pdyU,8432
-dcnum/read/__init__.py,sha256=ksLdV8EkOU3EPje8teCOSehcUeGAZfg9TQ5ltuEUgls,216
-dcnum/read/cache.py,sha256=lisrGG7AyvVitf0h92wh5FvYCsxa0pWyGcAyYwGP-LQ,6471
+dcnum/meta/ppid.py,sha256=RnDkJSdV1kDznAsOhQN5WI7uC9UwSMCjyADP7yWNvkM,8478
+dcnum/read/__init__.py,sha256=LYHyZHgiNTpjV5oEcty-7Kh5topLpHT_cFlNl-QX8gg,262
+dcnum/read/cache.py,sha256=LNA5nnDyrw8Nj07E7XfG2GcHEoWm6vA6Qo_8N-n-sGw,6492
 dcnum/read/const.py,sha256=GG9iyXDtEldvJYOBnhZjlimzIeBMAt4bSr2-xn2gzzc,464
-dcnum/read/hdf5_data.py,sha256=Yyq02UTILc5ZgIQXpR9Y0wuX2WT8s0g23PraI7KxmJY,23489
-dcnum/read/mapped.py,sha256=UryArlrIsHxjOyimBL2Nooi3r73zuGtnGdqdxa6PK_g,3076
+dcnum/read/detect_flicker.py,sha256=CeUyxI6LaX_lCNvBPm_yzsiWmiNcZYqbNZCtvKPdkcU,1827
+dcnum/read/hdf5_data.py,sha256=JVk9YWw1rPgTPxaMZsw2ehk4FJq9UqhmB1SW7yhPw50,25867
+dcnum/read/mapped.py,sha256=zU2fYdZfLNHn0rKHxDzBhNFMu4--WWa8nSeE2likyZA,3637
 dcnum/segm/__init__.py,sha256=9cLEAd3JWE8IGqDHV-eSDIYOGBfOepd8OcebtNs8Omk,309
 dcnum/segm/segm_thresh.py,sha256=iVhvIhzO0Gw0t3rXOgH71rOI0CNjJJQq4Gg6BulUhK8,948
 dcnum/segm/segmenter.py,sha256=FWLFDBR-x_85ku2rObA2F-QBrM4IUaUL-YHChLagVvM,14902
 dcnum/segm/segmenter_manager_thread.py,sha256=frM0sMxC7f7TQiFjmpRxuwG2kUBFpW1inV8dtpADHiI,5924
 dcnum/segm/segmenter_mpo.py,sha256=o6mQlITHgEWvQt9v6oCWwAcZUvxE7MOeLE9DFManzpY,13757
 dcnum/segm/segmenter_sto.py,sha256=e6MtN_RWusA0wTExV-FLGpDXNJs1CbSyXcSdWUPBMvM,3959
-dcnum/segm/segm_torch/__init__.py,sha256=re9jVLYvV1GgC7J5vx2LHKeFYVZPpiwubecAV9f_2kA,670
+dcnum/segm/segm_torch/__init__.py,sha256=DtUqJTbj7ybrTbXlwHq1Y4SCzi22rMW9Cus6wX-iU-A,822
 dcnum/segm/segm_torch/segm_torch_base.py,sha256=G9AhVyD6LkAmk0tkbYnJUSpvcj3_HYf0uqfILZQsyus,4479
-dcnum/segm/segm_torch/segm_torch_mpo.py,sha256=N01dVXai_4eIGfHJrPjg5C2Bkyq1TOeXeJhw3YbGidw,2504
+dcnum/segm/segm_torch/segm_torch_mpo.py,sha256=GOva6o-6_SppxWD4BeBB3ap1TR-6rIYHavtfIstaYvc,2643
 dcnum/segm/segm_torch/segm_torch_sto.py,sha256=PTOJrP_FkaxZZul8lM4VA2HL3KyxrheDDWWdJbmJdiw,3393
 dcnum/segm/segm_torch/torch_model.py,sha256=5aL6SwSvg1N2gATEGBhP3aA4WTHlvGzQVYuizmh0LrU,3187
 dcnum/segm/segm_torch/torch_postproc.py,sha256=ctirQTmsZnuZGIxkwFWN9arRneHRYJUxaJ_ZyCgjByM,3311
 dcnum/segm/segm_torch/torch_preproc.py,sha256=kjabu76paw23kO7RP7Ik6IY60Kk1VBAHKBAedflA0aQ,4002
-dcnum/write/__init__.py,sha256=QvWHeZmjHI18i-YlGYuzN3i7dVWY9UCReKchrJ-gif0,260
+dcnum/write/__init__.py,sha256=sK79IlvCFIqf2oFABVeyYedMnHOsEIQpxAauEeNO-Tw,273
 dcnum/write/deque_writer_thread.py,sha256=ao7F1yrVKyufgC4rC0Y2_Vt7snuT6KpI7W2qVxcjdhk,1994
-dcnum/write/queue_collector_thread.py,sha256=d_WfdsZdFnFsiAY0zVMwUlA4juIMeiWYmE_-rezBQCE,11734
-dcnum/write/writer.py,sha256=e6J8YVqhS7kzkpPIMoDMokJpqSy1WWNdOrwaJof1oVc,15601
-dcnum-0.23.1.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
-dcnum-0.23.1.dist-info/METADATA,sha256=NL79bLTjTqm5R4HW0TNcVZ9YXqbxH7Xcfl6xuk-hfw4,2280
-dcnum-0.23.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-dcnum-0.23.1.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
-dcnum-0.23.1.dist-info/RECORD,,
+dcnum/write/queue_collector_thread.py,sha256=KwwNIDFEF2DU83woKES5K05MxxOhDxPMZLLeyPugfDo,11542
+dcnum/write/writer.py,sha256=oHlq4bDHQxb33-3Fw1xnzJwACecLyH-6koGK8SN0cSk,20528
+dcnum-0.25.1.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
+dcnum-0.25.1.dist-info/METADATA,sha256=XtFOojvzsJHhAz_uxJW0vxg3mg3SYPB39tlKgLPGnbY,2280
+dcnum-0.25.1.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+dcnum-0.25.1.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
+dcnum-0.25.1.dist-info/RECORD,,

{dcnum-0.23.1.dist-info → dcnum-0.25.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (74.1.2)
 Root-Is-Purelib: true
 Tag: py3-none-any