dcnum 0.23.4__py3-none-any.whl → 0.24.0__py3-none-any.whl

dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.23.4'
-__version_tuple__ = version_tuple = (0, 23, 4)
+__version__ = version = '0.24.0'
+__version_tuple__ = version_tuple = (0, 24, 0)

dcnum/feat/feat_background/base.py

@@ -1,8 +1,10 @@
 import abc
 import functools
 import inspect
+import logging
 import multiprocessing as mp
 import pathlib
+import time
 
 import h5py
 
@@ -41,8 +43,11 @@ class Background(abc.ABC):
         kwargs:
             Additional keyword arguments passed to the subclass.
         """
+        self.logger = logging.getLogger(
+            f"dcnum.feat.feat_background.{self.__class__.__name__}")
         # proper conversion to Path objects
         output_path = pathlib.Path(output_path)
+        self.output_path = output_path
         if isinstance(input_data, str):
             input_data = pathlib.Path(input_data)
         # kwargs checks
@@ -188,20 +193,30 @@ class Background(abc.ABC):
             return self.image_proc.value
 
     def process(self):
+        """Perform the background computation
+
+        This irreversibly removes/overrides any "image_bg" and
+        "bg_off" features defined in the output file `self.h5out`.
+        """
+        t0 = time.perf_counter()
         # Delete any old background data
-        for key in ["image_bg", "bg_off"]:
-            if key in self.h5out["events"]:
-                del self.h5out["events"][key]
+        for ds_key in ["image_bg", "bg_off"]:
+            for grp_key in ["events", "basin_events"]:
+                if grp_key in self.h5out and ds_key in self.h5out[grp_key]:
+                    del self.h5out[grp_key][ds_key]
         # Perform the actual background computation
         self.process_approach()
         bg_ppid = self.get_ppid()
         # Store pipeline information in the image_bg/bg_off feature
-        for key in ["image_bg", "bg_off"]:
-            if key in self.h5out["events"]:
-                self.h5out[f"events/{key}"].attrs["dcnum ppid background"] = \
-                    bg_ppid
-                self.h5out[F"events/{key}"].attrs["dcnum ppid generation"] = \
-                    ppid.DCNUM_PPID_GENERATION
+        for ds_key in ["image_bg", "bg_off"]:
+            for grp_key in ["events", "basin_events"]:
+                if grp_key in self.h5out and ds_key in self.h5out[grp_key]:
+                    self.h5out[f"{grp_key}/{ds_key}"].attrs[
+                        "dcnum ppid background"] = bg_ppid
+                    self.h5out[F"{grp_key}/{ds_key}"].attrs[
+                        "dcnum ppid generation"] = ppid.DCNUM_PPID_GENERATION
+        self.logger.info(
+            f"Background computation time: {time.perf_counter()-t0:.1f}s")
 
     @abc.abstractmethod
     def process_approach(self):

dcnum/feat/feat_background/bg_sparse_median.py

@@ -1,4 +1,3 @@
-import logging
 import queue
 import time
 
@@ -9,14 +8,13 @@ from ...read import HDF5Data
 
 from .base import mp_spawn, Background
 
-logger = logging.getLogger(__name__)
-
 
 class BackgroundSparseMed(Background):
     def __init__(self, input_data, output_path, kernel_size=200,
                  split_time=1., thresh_cleansing=0, frac_cleansing=.8,
                  offset_correction=True,
-                 compress=True, num_cpus=None):
+                 compress=True,
+                 num_cpus=None):
         """Sparse median background correction with cleansing
 
         In contrast to the rolling median background correction,
@@ -79,6 +77,11 @@ class BackgroundSparseMed(Background):
         num_cpus: int
             Number of CPUs to use for median computation. Defaults to
             `multiprocessing.cpu_count()`.
+
+        .. versionchanged:: 0.23.5
+
+            The background image data are stored as an internal
+            mapped basin to reduce the output file size.
         """
         super(BackgroundSparseMed, self).__init__(
             input_data=input_data,
@@ -93,7 +96,7 @@ class BackgroundSparseMed(Background):
         )
 
         if kernel_size > len(self.input_data):
-            logger.warning(
+            self.logger.warning(
                 f"The kernel size {kernel_size} is too large for input data"
                 f"size {len(self.input_data)}. Setting it to input data size!")
             kernel_size = len(self.input_data)
@@ -126,13 +129,14 @@ class BackgroundSparseMed(Background):
                 else:
                     # compute time using frame rate (approximate)
                     dur = self.image_count / fr * 1.5
-                    logger.info(f"Approximating duration: {dur/60:.1f}min")
+                    self.logger.info(
+                        f"Approximating duration: {dur/60:.1f}min")
                     self.time = np.linspace(0, dur, self.image_count,
                                             endpoint=True)
         if self.time is None:
             # No HDF5 file or no information therein; Make an educated guess.
             dur = self.image_count / 3600 * 1.5
-            logger.info(f"Guessing duration: {dur/60:.1f}min")
+            self.logger.info(f"Guessing duration: {dur/60:.1f}min")
             self.time = np.linspace(0, dur, self.image_count,
                                     endpoint=True)
 
@@ -301,18 +305,18 @@ class BackgroundSparseMed(Background):
                 thresh = np.quantile(ref, self.frac_cleansing)
                 used = ref <= thresh
                 frac_remove = np.sum(~used) / used.size
-                logger.warning(
+                self.logger.warning(
                     f"{frac_remove_user:.1%} of the background images would "
                     f"be removed with the current settings, so we enforce "
                     f"`frac_cleansing`. To avoid this warning, try decreasing "
                     f"`thresh_cleansing` or `frac_cleansing`. The new "
                     f"threshold is {thresh_fact / thresh}.")
 
-            logger.info(f"Cleansed {frac_remove:.2%}")
+            self.logger.info(f"Cleansed {frac_remove:.2%}")
             step_times = self.step_times[used]
             bg_images = self.bg_images[used]
         else:
-            logger.info("Background series cleansing disabled")
+            self.logger.info("Background series cleansing disabled")
             step_times = self.step_times
             bg_images = self.bg_images
 
@@ -322,35 +326,55 @@ class BackgroundSparseMed(Background):
         idx1 = None
         for ii in range(len(step_times)):
             t1 = step_times[ii]
-            idx1 = np.argmin(np.abs(self.time - t1 + self.split_time/2))
+            idx1 = np.argmin(np.abs(self.time - t1 - self.split_time/2))
             bg_idx[idx0:idx1] = ii
             idx0 = idx1
         if idx1 is not None:
             # Fill up remainder of index array with last entry
             bg_idx[idx1:] = ii
 
-        self.image_proc.value = 1
-
-        # Write background data
-        pos = 0
-        step = 1000
-        while pos < self.image_count:
-            stop = min(pos + step, self.image_count)
-            cur_slice = slice(pos, stop)
-            cur_bg_data = bg_images[bg_idx[cur_slice]]
-            self.writer.store_feature_chunk("image_bg", cur_bg_data)
-            if self.offset_correction:
+        # Store the background images as an internal mapped basin
+        self.writer.store_basin(
+            name="background images",
+            description=f"Pipeline identifier: {self.get_ppid()}",
+            mapping=bg_idx,
+            internal_data={"image_bg": bg_images}
+            )
+
+        # store the offset correction, if applicable
+        if self.offset_correction:
+            self.logger.info("Computing offset correction")
+            # compute the mean at the top of all background images
+            sh, sw = self.input_data.shape[1:]
+            roi_full = (slice(None), slice(0, 20), slice(0, sw))
+            bg_data_mean = np.mean(bg_images[roi_full], axis=(1, 2))
+            pos = 0
+            step = self.writer.get_best_nd_chunks(item_shape=(sh, sw),
+                                                  feat_dtype=np.uint8)[0]
+            bg_off = np.zeros(self.image_count, dtype=float)
+            # For every chunk in the input image data, compute that
+            # value as well and store the resulting offset value.
+            # TODO: Could this be parallelized, or are we limited in reading?
+            while pos < self.image_count:
+                stop = min(pos + step, self.image_count)
                 # Record background offset correction "bg_off". We take a
                 # slice of 20px from the top of the image (there are normally
                 # no events here, only the channel walls are visible).
-                sh, sw = self.input_data.shape[1:]
-                roi_full = (slice(None), slice(0, 20), slice(0, sw))
+                cur_slice = slice(pos, stop)
+                # mean background brightness
+                val_bg = bg_data_mean[bg_idx[cur_slice]]
+                # mean image brightness
                 roi_cur = (cur_slice, slice(0, 20), slice(0, sw))
-                val_bg = np.mean(cur_bg_data[roi_full], axis=(1, 2))
                 val_dat = np.mean(self.input_data[roi_cur], axis=(1, 2))
                 # background image = image_bg + bg_off
-                self.writer.store_feature_chunk("bg_off", val_dat - val_bg)
-            pos += step
+                bg_off[cur_slice] = val_dat - val_bg
+                # set progress
+                self.image_proc.value = 0.5 * (1 + pos / self.image_count)
+                pos = stop
+            # finally, store the background offset feature
+            self.writer.store_feature_chunk("bg_off", bg_off)
+
+        self.image_proc.value = 1
 
     def process_second(self,
                        ii: int,
@@ -393,7 +417,9 @@ class BackgroundSparseMed(Background):
 
         self.bg_images[ii] = self.shared_output.reshape(self.image_shape)
 
-        self.image_proc.value = idx_stop / self.image_count
+        self.image_proc.value = idx_stop / (
+                # with offset correction, everything is slower
+                self.image_count * (1 + self.offset_correction))
 
 
 class WorkerSparseMed(mp_spawn.Process):

dcnum/logic/ctrl.py

@@ -403,6 +403,12 @@ class DCNumJobRunner(threading.Thread):
                                   features=orig_feats,
                                   mapping=None)
 
+        # Handle basin data according to the user's request
+        self.state = "plumbing"
+        self.task_enforce_basin_strategy()
+
+        self.state = "cleanup"
+
         with HDF5Writer(self.path_temp_out) as hw:
             # pipeline metadata
             hw.h5.attrs["pipeline:dcnum generation"] = self.ppdict["gen_id"]
@@ -462,11 +468,7 @@ class DCNumJobRunner(threading.Thread):
 
             # copy metadata/logs/tables from original file
             with h5py.File(self.job["path_in"]) as h5_src:
-                copy_metadata(h5_src=h5_src,
-                              h5_dst=hw.h5,
-                              # Don't copy basins, we would have to index-map
-                              # them first.
-                              copy_basins=False)
+                copy_metadata(h5_src=h5_src, h5_dst=hw.h5)
             if redo_seg:
                 # Store the correct measurement identifier. This is used to
                 # identify this file as a correct basin in subsequent pipeline
@@ -490,12 +492,6 @@ class DCNumJobRunner(threading.Thread):
                 mid_new = f"{mid_cur}_{mid_ap}" if mid_cur else mid_ap
                 hw.h5.attrs["experiment:run identifier"] = mid_new
 
-        # Handle basin data according to the user's request
-        self.state = "plumbing"
-        self.task_enforce_basin_strategy()
-
-        self.state = "cleanup"
-
         trun = datetime.timedelta(seconds=round(time.monotonic() - time_start))
         self.logger.info(f"Run duration: {str(trun)}")
         self.logger.info(time.strftime("Run stop: %Y-%m-%d-%H.%M.%S",
@@ -547,22 +543,17 @@ class DCNumJobRunner(threading.Thread):
         """
         self._progress_bn = 0
         t0 = time.perf_counter()
-        # We need to make sure that the features are correctly attributed
-        # from the input files. E.g. if the input file already has
-        # background images, but we recompute the background images, then
-        # we have to use the data from the recomputed background file.
-        # We achieve this by keeping a specific order and only copying those
-        # features that we don't already have in the output file.
-        feats_raw = [
-            # 1. background data from the temporary input image
-            # (this must come before draw [sic!])
-            [self.dtin.h5, ["image_bg", "bg_off"], "critical"],
-            # 2. frame-based scalar features from the raw input file
-            # (e.g. "temp" or "frame")
-            [self.draw.h5, self.draw.features_scalar_frame, "optional"],
-            # 3. image features from the input file
-            [self.draw.h5, ["image", "image_bg", "bg_off"], "optional"],
-        ]
+        # We have these points to consider:
+        # - We must use the `basinmap` feature to map from the original
+        #   file to the output file.
+        # - We must copy "bg_off" and "image_bg" to the output file.
+        # - For the "drain" basin strategy, we also have to copy all the
+        #   other features.
+        # - If "image_bg" is defined as an internal basin in the input
+        #   file, we have to convert the mapping and store a corresponding
+        #   internal basin in the output file.
+
+        # Determine the basinmap feature
         with HDF5Writer(self.path_temp_out) as hw:
             hout = hw.h5
             # First, we have to determine the basin mapping from input to
@@ -584,14 +575,15 @@ class DCNumJobRunner(threading.Thread):
                 # to the original input HDF5 file.
                 raw_im = self.draw.index_mapping
                 if raw_im is None:
-                    self.logger.info("Input file mapped with basinmap0")
                     # Create a hard link to save time and space
                     hout["events/basinmap0"] = hout["events/index_unmapped"]
-                    basinmap = idx_um
+                    basinmap0 = idx_um
                 else:
-                    basinmap = get_mapping_indices(raw_im)[idx_um]
+                    self.logger.info("Converting input mapping")
+                    basinmap0 = get_mapping_indices(raw_im)[idx_um]
                     # Store the mapped basin data in the output file.
-                    hw.store_feature_chunk("basinmap0", basinmap)
+                    hw.store_feature_chunk("basinmap0", basinmap0)
+                self.logger.info("Input mapped to output with basinmap0")
                 # We don't need them anymore.
                 del hout["events/index_unmapped"]
 
@@ -599,19 +591,72 @@ class DCNumJobRunner(threading.Thread):
                 # is the size of the raw dataset and the latter is its mapped
                 # size!
                 size_raw = self.draw.h5.attrs["experiment:event count"]
-                if (len(basinmap) == size_raw
-                        and np.all(basinmap == np.arange(size_raw))):
+                if (len(basinmap0) == size_raw
+                        and np.all(basinmap0 == np.arange(size_raw))):
                     # This means that the images in the input overlap perfectly
                     # with the images in the output, i.e. a "copy" segmenter
                     # was used or something is very reproducible.
                     # We set basinmap to None to be more efficient.
-                    basinmap = None
+                    basinmap0 = None
 
             else:
                 # The input is identical to the output, because we are using
                 # the same pipeline identifier.
-                basinmap = None
-
+                basinmap0 = None
+
+            # List of features we have to copy from input to output.
+            # We need to make sure that the features are correctly attributed
+            # from the input files. E.g. if the input file already has
+            # background images, but we recompute the background images, then
+            # we have to use the data from the recomputed background file.
+            # We achieve this by keeping a specific order and only copying
+            # those features that we don't already have in the output file.
+            feats_raw = [
+                # background data from the temporary input image
+                [self.dtin.h5, ["bg_off"], "critical"],
+                [self.draw.h5, self.draw.features_scalar_frame, "optional"],
+                [self.draw.h5, ["image", "bg_off"], "optional"],
+            ]
+
+            # Store image_bg as an internal basin, if defined in input
+            for idx in range(len(self.dtin.basins)):
+                bn_dict = self.dtin.basins[idx]
+                if (bn_dict["type"] == "internal"
+                        and "image_bg" in bn_dict["features"]):
+                    self.logger.info(
+                        "Copying internal basin background images")
+                    bn_grp, bn_feats, bn_map = self.dtin.get_basin_data(idx)
+                    assert "image_bg" in bn_feats
+                    # Load all images into memory (should only be ~600)
+                    bg_images1 = self.dtin.h5["basin_events"]["image_bg"][:]
+                    # Get the original internal mapping for these images
+                    # Note that `basinmap0` always refers to indices in the
+                    # original raw input file, and not to indices in an
+                    # optional mapped input file (using `index_mapping`).
+                    # Therefore, we do `self.dtin.h5["events"]["basinmap0"]`
+                    # instead of `self.dtin["basinmap0"]`
+                    basinmap_in = self.dtin.h5["events"][bn_dict["mapping"]][:]
+                    # Now we have to convert the indices in `basinmap_in`
+                    # to indices in the output file.
+                    basinmap1 = basinmap_in[basinmap0]
+                    # Store the internal mapping in the output file
+                    hw.store_basin(name=bn_dict["name"],
+                                   description=bn_dict["description"],
+                                   mapping=basinmap1,
+                                   internal_data={"image_bg": bg_images1}
+                                   )
+                    break
+            else:
+                self.logger.info("Background images must be copied")
+                # There is no internal image_bg feature, probably because
+                # the user did not use the sparsemed background correction.
+                # In this case, we simply add "image_bg" to the `feats_raw`.
+                feats_raw += [
+                    [self.dtin.h5, ["image_bg"], "critical"],
+                    [self.draw.h5, ["image_bg"], "optional"],
+                ]
+
+            # Copy the features required in the output file.
             for hin, feats, importance in feats_raw:
                 # Only consider features that are available in the input
                 # and that are not already in the output.
@@ -626,7 +671,7 @@ class DCNumJobRunner(threading.Thread):
                     copy_features(h5_src=hin,
                                   h5_dst=hout,
                                   features=feats,
-                                  mapping=basinmap)
+                                  mapping=basinmap0)
                 else:
                     # TAP: Create basins for the "optional" features in the
                     # output file. Note that the "critical" features never
@@ -638,7 +683,7 @@ class DCNumJobRunner(threading.Thread):
                     paths = [pin, os.path.relpath(pin, pout)]
                     hw.store_basin(name="dcnum basin",
                                    features=feats,
-                                   mapping=basinmap,
+                                   mapping=basinmap0,
                                    paths=paths,
                                    description=f"Created with dcnum {version}",
                                    )

dcnum/meta/ppid.py

@@ -10,7 +10,7 @@ import warnings
 
 #: Increment this string if there are breaking changes that make
 #: previous pipelines unreproducible.
-DCNUM_PPID_GENERATION = "10"
+DCNUM_PPID_GENERATION = "11"
 
 
 class ClassWithPPIDCapabilities(Protocol):

dcnum/read/hdf5_data.py

@@ -102,10 +102,12 @@ class HDF5Data:
         elif (feat in self.h5["events"]
               and len(self.h5["events"][feat].shape) == 1):  # cache scalar
             if self.index_mapping is None:
-                idx_map = slice(None)  # no mapping indices, just slice
+                # no mapping indices, just slice
+                dat_sc = self.h5["events"][feat][:]
             else:
-                idx_map = get_mapping_indices(self.index_mapping)
-            self._cache_scalar[feat] = self.h5["events"][feat][idx_map]
+                dat_sc = get_mapped_object(self.h5["events"][feat],
+                                           index_mapping=self.index_mapping)[:]
+            self._cache_scalar[feat] = dat_sc
             return self._cache_scalar[feat]
         else:
             if feat in self.h5["events"]:
@@ -117,9 +119,11 @@ class HDF5Data:
             else:
                 # Check the basins
                 for idx in range(len(self.basins)):
-                    bn, bn_features = self.get_basin_data(idx)
-                    if bn_features and feat in bn_features:
-                        return bn[feat]  # already index-mapped
+                    bn_grp, bn_feats, bn_map = self.get_basin_data(idx)
+                    if bn_feats and feat in bn_feats:
+                        mapped_ds = get_mapped_object(obj=bn_grp[feat],
+                                                      index_mapping=bn_map)
+                        return mapped_ds
         # If we got here, then the feature data does not exist.
         raise KeyError(f"Feature '{feat}' not found in {self}!")
 
@@ -200,14 +204,7 @@ class HDF5Data:
                             np.array(h5["tables"][tab][tkey]).reshape(-1)
                     self.tables[tab] = tabdict
                 # basins
-                basins = []
-                for bnkey in h5.get("basins", {}).keys():
-                    bn_data = "\n".join(
-                        [s.decode() for s in h5["basins"][bnkey][:].tolist()])
-                    bn_dict = json.loads(bn_data)
-                    if bn_dict["type"] == "file":
-                        # we only support file-based basins
-                        basins.append(bn_dict)
+                basins = self.extract_basin_dicts(h5)
                 self.basins = sorted(basins, key=lambda x: x["name"])
 
         if state["pixel_size"] is not None:
@@ -273,6 +270,30 @@ class HDF5Data:
         pixel_size = float(f"{pixel_size:.8f}")
         self.meta["imaging:pixel size"] = pixel_size
 
+    @staticmethod
+    def extract_basin_dicts(h5, check=True):
+        """Return list of basin dictionaries"""
+        # TODO:
+        #  - support iterative mapped basins and catch
+        #    circular basin definitions.
+        basins = []
+        for bnkey in h5.get("basins", {}).keys():
+            bn_data = "\n".join(
+                [s.decode() for s in h5["basins"][bnkey][:].tolist()])
+            bn_dict = json.loads(bn_data)
+            if check:
+                if bn_dict["type"] not in ["internal", "file"]:
+                    # we only support file-based and internal basins
+                    continue
+                basinmap = bn_dict.get("mapping")
+                if basinmap is not None and basinmap not in h5["events"]:
+                    # basinmap feature is missing
+                    continue
+            # Add the basin
+            basins.append(bn_dict)
+
+        return basins
+
     @property
     def features_scalar_frame(self):
         """Scalar features that apply to all events in a frame
@@ -291,9 +312,10 @@ class HDF5Data:
 
     def close(self):
         """Close the underlying HDF5 file"""
-        for bn, _ in self._basin_data.values():
-            if bn is not None:
-                bn.close()
+        for bn_group, _, _ in self._basin_data.values():
+            if bn_group is not None:
+                if bn_group.id.valid:
+                    bn_group.file.close()
         self._image_cache.clear()
         self._basin_data.clear()
         self.h5.close()
@@ -371,66 +393,110 @@ class HDF5Data:
                 raise ValueError(f"Invalid parameter '{var}'!")
         return kwargs
 
-    def get_basin_data(self, index):
+    def get_basin_data(self, index: int) -> (
+            h5py.Group,
+            List,
+            int | slice | List | np.ndarray,
+            ):
         """Return HDF5Data info for a basin index in `self.basins`
 
+        Parameters
+        ----------
+        index: int
+            index of the basin from which to get data
+
         Returns
         -------
-        data: HDF5Data
-            Data instance
+        group: h5py.Group
+            HDF5 group containing HDF5 Datasets with the names
+            listed in `features`
         features: list of str
-            List of features made available by this data instance
+            list of features made available by this basin
+        index_mapping:
+            a mapping (see `__init__`) that defines mapping from
+            the basin dataset to the referring dataset
         """
         if index not in self._basin_data:
             bn_dict = self.basins[index]
-            for ff in bn_dict["paths"]:
-                pp = pathlib.Path(ff)
-                if pp.is_absolute() and pp.exists():
-                    path = pp
-                    break
-                else:
-                    # try relative path
-                    prel = pathlib.Path(self.path).parent / pp
-                    if prel.exists():
-                        path = prel
-                        break
+
+            # HDF5 group containing the feature data
+            if bn_dict["type"] == "file":
+                h5group, features = self._get_basin_data_file(bn_dict)
+            elif bn_dict["type"] == "internal":
+                h5group, features = self._get_basin_data_internal(bn_dict)
             else:
-                path = None
-            if path is None:
-                self._basin_data[index] = (None, None)
+                raise ValueError(f"Invalid basin type '{bn_dict['type']}'")
+
+            # index mapping
+            feat_basinmap = bn_dict.get("mapping", None)
+            if feat_basinmap is None:
+                # This is NOT a mapped basin.
+                index_mapping = self.index_mapping
             else:
-                feat_basinmap = bn_dict.get("mapping", None)
-                if feat_basinmap is None:
-                    # This is NOT a mapped basin.
-                    index_mapping = self.index_mapping
+                # This is a mapped basin. Create an indexing list.
+                if self.index_mapping is None:
+                    # The current dataset is not mapped.
+                    basinmap_idx = slice(None)
                 else:
-                    # This is a mapped basin. Create an indexing list.
-                    if self.index_mapping is None:
-                        # The current dataset is not mapped.
-                        basinmap_idx = slice(None)
-                    else:
-                        # The current dataset is also mapped.
-                        basinmap_idx = get_mapping_indices(self.index_mapping)
-                    basinmap = self.h5[f"events/{feat_basinmap}"]
-                    index_mapping = basinmap[basinmap_idx]
-
-                h5dat = HDF5Data(path, index_mapping=index_mapping)
-                features = bn_dict.get("features")
-                if features is None:
-                    # Only get the features from the actual HDF5 file.
-                    # If this file has basins as well, the basin metadata
-                    # should have been copied over to the parent file. This
-                    # makes things a little cleaner, because basins are not
-                    # nested, but all basins are available in the top file.
-                    # See :func:`write.store_metadata` for copying metadata
-                    # between files.
-                    # The writer can still specify "features" in the basin
-                    # metadata, then these basins are indeed nested, and
-                    # we consider that ok as well.
-                    features = sorted(h5dat.h5["events"].keys())
-                self._basin_data[index] = (h5dat, features)
+                    # The current dataset is also mapped.
+                    basinmap_idx = get_mapping_indices(self.index_mapping)
+                basinmap = self.h5[f"events/{feat_basinmap}"]
+                index_mapping = basinmap[basinmap_idx]
+
+            self._basin_data[index] = (h5group, features, index_mapping)
         return self._basin_data[index]
 
+    def _get_basin_data_file(self, bn_dict):
+        for ff in bn_dict["paths"]:
+            pp = pathlib.Path(ff)
+            if pp.is_absolute() and pp.exists():
+                path = pp
+                break
+            else:
+                # try relative path
+                prel = pathlib.Path(self.path).parent / pp
+                if prel.exists():
+                    path = prel
+                    break
+        else:
+            path = None
+        if path is None:
+            # Cannot get data from this basin / cannot find file
+            h5group = None
+            features = []
+        else:
+            h5 = h5py.File(path, "r")
+            h5group = h5["events"]
+            # features defined in the basin
+            features = bn_dict.get("features")
+            if features is None:
+                # Only get the features from the actual HDF5 file.
+                # If this file has basins as well, the basin metadata
+                # should have been copied over to the parent file. This
+                # makes things a little cleaner, because basins are not
+                # nested, but all basins are available in the top file.
+                # See :func:`write.store_metadata` for copying metadata
+                # between files.
+                # The writer can still specify "features" in the basin
+                # metadata, then these basins are indeed nested, and
+                # we consider that ok as well.
+                features = sorted(h5group.keys())
+        return h5group, features
+
+    def _get_basin_data_internal(self, bn_dict):
+        # The group name is normally "basin_events"
+        group_name = bn_dict["paths"][0]
+        if group_name != "basin_events":
+            warnings.warn(
+                f"Uncommon group name for basin features: {group_name}")
+        h5group = self.h5[group_name]
+        features = bn_dict.get("features")
+        if features is None:
+            raise ValueError(
+                f"Encountered invalid internal basin '{bn_dict}': "
+                f"'features' must be defined")
+        return h5group, features
+
     def get_image_cache(self, feat):
         """Create an HDF5ImageCache object for the current dataset
 
@@ -444,15 +510,15 @@ class HDF5Data:
                 idx_map = None
                 # search all basins
                 for idx in range(len(self.basins)):
-                    bn_dat, features = self.get_basin_data(idx)
-                    if features is not None:
-                        if feat in features:
+                    bn_grp, bn_feats, bn_map = self.get_basin_data(idx)
+                    if bn_feats is not None:
+                        if feat in bn_feats:
                             # HDF5 dataset
-                            ds = bn_dat.h5[f"events/{feat}"]
+                            ds = bn_grp[feat]
                             # Index mapping (taken from the basins which
                             # already includes the mapping from the current
                             # instance).
-                            idx_map = bn_dat.index_mapping
+                            idx_map = bn_map
                             break
                 else:
                     ds = None
@@ -473,9 +539,9 @@ class HDF5Data:
             features = sorted(self.h5["/events"].keys())
             # add basin features
             for ii in range(len(self.basins)):
-                _, bfeats = self.get_basin_data(ii)
-                if bfeats:
-                    features += bfeats
+                _, bn_feats, _ = self.get_basin_data(ii)
+                if bn_feats:
+                    features += bn_feats
             self._keys = sorted(set(features))
         return self._keys
 

dcnum/read/mapped.py

@@ -27,8 +27,21 @@ class MappedHDF5Dataset:
         if isinstance(idx, numbers.Integral):
             return self.h5ds[self.mapping_indices[idx]]
         else:
-            idx_mapped = self.mapping_indices[idx]
-            return self.h5ds[idx_mapped]
+            midx = self.mapping_indices[idx]
+            start = np.min(midx)
+            # Add one, because the final index must be included
+            stop = np.max(midx) + 1
+            # We have to perform mapping.
+            # Since h5py is very slow at indexing with arrays,
+            # we instead read the data in chunks from the input file,
+            # and perform the mapping afterward using the numpy arrays.
+            data_in = self.h5ds[start:stop]
+            # Determine the indices that we need from that chunk.
+            data = data_in[midx - start]
+            return data
+
+    def __len__(self):
+        return self.shape[0]
 
 
 def get_mapping_indices(

dcnum/write/__init__.py

@@ -2,5 +2,5 @@
 from .deque_writer_thread import DequeWriterThread
 from .queue_collector_thread import EventStash, QueueCollectorThread
 from .writer import (
-    HDF5Writer, copy_features, copy_metadata, create_with_basins,
+    HDF5Writer, copy_basins, copy_features, copy_metadata, create_with_basins,
     set_default_filter_kwargs)

dcnum/write/writer.py

@@ -1,13 +1,14 @@
 import hashlib
 import json
 import pathlib
-from typing import Dict, List
+from typing import Dict, List, Tuple
 import warnings
 
 import h5py
 import hdf5plugin
 import numpy as np
 
+from ..read import HDF5Data
 from .._version import version
 
 
@@ -16,6 +17,11 @@ class CreatingFileWithoutBasinWarning(UserWarning):
     pass
 
 
+class IgnoringBasinTypeWarning(UserWarning):
+    """Issued when a specific basin type is ignored"""
+    pass
+
+
 class HDF5Writer:
     def __init__(self,
                  # TODO: make this a mandatory argument when `path` is
@@ -90,14 +96,42 @@ class HDF5Writer:
         chunk_size_int = max(10, int(np.floor(chunk_size)))
         return tuple([chunk_size_int] + list(item_shape))
 
-    def require_feature(self, feat, item_shape, feat_dtype, ds_kwds=None):
-        """Create a new feature in the "events" group"""
-        if feat not in self.events:
+    def require_feature(self,
+                        feat: str,
+                        item_shape: Tuple[int],
+                        feat_dtype: np.dtype,
+                        ds_kwds: Dict = None,
+                        group_name: str = "events"):
+        """Create a new feature in the "events" group
+
+        Parameters
+        ----------
+        feat: str
+            name of the feature
+        item_shape: Tuple[int]
+            shape for one event of this feature, e.g. for a scalar
+            event, the shape would be `(1,)` and for an image, the
+            shape could be `(80, 300)`.
+        feat_dtype: np.dtype
+            dtype of the feature
+        ds_kwds: Dict
+            HDF5 Dataset keyword arguments (e.g. compression, fletcher32)
+        group_name: str
+            name of the HDF5 group where the feature should be written to;
+            defaults to the "events" group, but a different group can be
+            specified for storing e.g. internal  basin features.
+        """
+        if group_name == "events":
+            egroup = self.events
+        else:
+            egroup = self.h5.require_group(group_name)
+
+        if feat not in egroup:
             if ds_kwds is None:
                 ds_kwds = {}
             for key in self.ds_kwds:
                 ds_kwds.setdefault(key, self.ds_kwds[key])
-            dset = self.events.create_dataset(
+            dset = egroup.create_dataset(
                 feat,
                 shape=tuple([0] + list(item_shape)),
                 dtype=feat_dtype,
@@ -112,16 +146,17 @@ class HDF5Writer:
                                   np.string_('IMAGE_GRAYSCALE'))
             offset = 0
         else:
-            dset = self.events[feat]
+            dset = egroup[feat]
             offset = dset.shape[0]
         return dset, offset
 
     def store_basin(self,
                     name: str,
-                    paths: List[str | pathlib.Path],
+                    paths: List[str | pathlib.Path] | None = None,
                     features: List[str] = None,
                     description: str | None = None,
-                    mapping: np.ndarray = None
+                    mapping: np.ndarray = None,
+                    internal_data: Dict | None = None,
                     ):
         """Write an HDF5-based file basin
 
@@ -129,8 +164,9 @@ class HDF5Writer:
         ----------
         name: str
             basin name; Names do not have to be unique.
-        paths: list of str or pathlib.Path
-            location(s) of the basin
+        paths: list of str or pathlib.Path or None
+            location(s) of the basin; must be None when storing internal
+            data, a list of paths otherwise
         features: list of str
             list of features provided by `paths`
         description: str
@@ -138,14 +174,39 @@ class HDF5Writer:
         mapping: 1D array
             integer array with indices that map the basin dataset
             to this dataset
+        internal_data: dict of ndarrays
+            internal basin data to store; If this is set, then `features`
+            and `paths` must be set to `None`.
         """
         bdat = {
             "description": description,
-            "format": "hdf5",
             "name": name,
-            "paths": [str(pp) for pp in paths],
-            "type": "file",
         }
+
+        if internal_data:
+            if features is not None:
+                raise ValueError("`features` must be set to None when storing "
+                                 "internal basin features")
+            if paths is not None:
+                raise ValueError("`paths` must be set to None when storing "
+                                 "internal basin features")
+            # store the internal basin information
+            for feat in internal_data:
+                if feat in self.h5.require_group("basin_events"):
+                    raise ValueError(f"Feature '{feat}' is already defined "
+                                     f"as an internal basin feature")
+                self.store_feature_chunk(feat=feat,
+                                         data=internal_data[feat],
+                                         group_name="basin_events")
+            features = sorted(internal_data.keys())
+            bdat["format"] = "h5dataset"
+            bdat["paths"] = ["basin_events"]
+            bdat["type"] = "internal"
+        else:
+            bdat["format"] = "hdf5"
+            bdat["paths"] = [str(pp) for pp in paths]
+            bdat["type"] = "file"
+
         # Explicit features stored in basin file
         if features is not None and len(features):
             bdat["features"] = features
@@ -195,7 +256,7 @@ class HDF5Writer:
                 chunks=True,
                 **self.ds_kwds)
 
-    def store_feature_chunk(self, feat, data):
+    def store_feature_chunk(self, feat, data, group_name="events"):
         """Store feature data
 
         The "chunk" implies that always chunks of data are stored,
@@ -205,7 +266,8 @@ class HDF5Writer:
             data = 255 * np.array(data, dtype=np.uint8)
         ds, offset = self.require_feature(feat=feat,
                                           item_shape=data.shape[1:],
-                                          feat_dtype=data.dtype)
+                                          feat_dtype=data.dtype,
+                                          group_name=group_name)
         dsize = data.shape[0]
         ds.resize(offset + dsize, axis=0)
         ds[offset:offset + dsize] = data
@@ -291,14 +353,17 @@ def create_with_basins(
                 # copy metadata
                 with h5py.File(prep, libver="latest") as h5:
                     copy_metadata(h5_src=h5, h5_dst=hw.h5)
+                    copy_basins(h5_src=h5, h5_dst=hw.h5)
                     # extract features
                     features = sorted(h5["events"].keys())
+                    features = [f for f in features if
+                                not f.startswith("basinmap")]
                 name = prep.name
             else:
                 features = None
                 name = bps[0]
 
-            # Finally, write the basin.
+            # Write the basin data
             hw.store_basin(name=name,
                            paths=bps,
                            features=features,
@@ -306,6 +371,44 @@ def create_with_basins(
                            )
 
 
+def copy_basins(h5_src: h5py.File,
+                h5_dst: h5py.File,
+                internal_basins: bool = True
+                ):
+    """Reassemble basin data in the output file
+
+    This does not just copy the datasets defined in the "basins"
+    group, but it also loads the "basinmap?" features and stores
+    them as new "basinmap?" features in the output file.
+    """
+    basins = HDF5Data.extract_basin_dicts(h5_src, check=False)
+    hw = HDF5Writer(h5_dst)
+    for bn_dict in basins:
+        if bn_dict["type"] == "internal" and internal_basins:
+            internal_data = {}
+            for feat in bn_dict["features"]:
+                internal_data[feat] = h5_src["basin_events"][feat]
+            hw.store_basin(name=bn_dict["name"],
+                           description=bn_dict["description"],
+                           mapping=h5_src["events"][bn_dict["mapping"]][:],
+                           internal_data=internal_data,
+                           )
+        elif bn_dict["type"] == "file":
+            if bn_dict.get("mapping") is not None:
+                mapping = h5_src["events"][bn_dict["mapping"]][:]
+            else:
+                mapping = None
+            hw.store_basin(name=bn_dict["name"],
+                           description=bn_dict["description"],
+                           paths=bn_dict["paths"],
+                           features=bn_dict["features"],
+                           mapping=mapping,
+                           )
+        else:
+            warnings.warn(f"Ignored basin of type '{bn_dict['type']}'",
+                          IgnoringBasinTypeWarning)
+
+
 def copy_features(h5_src: h5py.File,
                   h5_dst: h5py.File,
                   features: List[str],
@@ -370,9 +473,9 @@ def copy_features(h5_src: h5py.File,
 
 
 def copy_metadata(h5_src: h5py.File,
-                  h5_dst: h5py.File,
-                  copy_basins=True):
-    """Copy attributes, tables, logs, and basins from one H5File to another
+                  h5_dst: h5py.File
+                  ):
+    """Copy attributes, tables, and logs from one H5File to another
 
     Notes
     -----
@@ -386,8 +489,6 @@ def copy_metadata(h5_src: h5py.File,
     for kk in src_attrs:
         h5_dst.attrs.setdefault(kk, src_attrs[kk])
     copy_data = ["logs", "tables"]
-    if copy_basins:
-        copy_data.append("basins")
     # copy other metadata
     for topic in copy_data:
         if topic in h5_src:

{dcnum-0.23.4.dist-info → dcnum-0.24.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.23.4
+Version: 0.24.0
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.23.4.dist-info → dcnum-0.24.0.dist-info}/RECORD

@@ -1,14 +1,14 @@
 dcnum/__init__.py,sha256=hcawIKS7utYiOyVhOAX9t7K3xYzP1b9862VV0b6qSrQ,74
-dcnum/_version.py,sha256=tQn7-kBQAh7UWs8qkwK03x5s3IM93ZqMm06Z5CYzTWI,413
+dcnum/_version.py,sha256=DKJfCmnOSzDR_ntzwRoytwuILw8hpyVnYbxMRkBXRw4,413
 dcnum/feat/__init__.py,sha256=jUJYWTD3VIoDNKrmryXbjHb1rGwYtK4b7VPWihYgUoo,325
 dcnum/feat/event_extractor_manager_thread.py,sha256=FAxSyRfaNAuBWNplxHngp5h-44s0qIP24XX_oETdfMk,7836
 dcnum/feat/gate.py,sha256=Yhxq80JoRMmQzBxl35C8NT91c9QcmQa-EIKLuxK6WvE,7221
 dcnum/feat/queue_event_extractor.py,sha256=0ncTQleT1sfc98zYkFuZWxU-akecfTrW6-OOU3z-d8o,15698
 dcnum/feat/feat_background/__init__.py,sha256=OTmMuazHNaSrZb2XW4cnJ6PlgJLbKrPbaidpEixYa0A,341
-dcnum/feat/feat_background/base.py,sha256=A-K3qlJ0ABFBGm5eMKYcNCC7ktFAInSm0eR3N3DHQZY,7963
+dcnum/feat/feat_background/base.py,sha256=bQBPvztrku-8YSVk8YBUUNh7MaYcnztgyD2-dQHxpzw,8674
 dcnum/feat/feat_background/bg_copy.py,sha256=PK8x4_Uph-_A6uszZC5uhe1gD1dSRdHnDMEsN0HSGHA,1034
 dcnum/feat/feat_background/bg_roll_median.py,sha256=EyjstMDXFBYuJB1lN6g4Uw7tPm434X3hXQxKSqvcoJ4,13175
-dcnum/feat/feat_background/bg_sparse_median.py,sha256=ab7Boj7cmr6PBdTbyWTj_yNNJSfuowr7u-iSGW989WI,20709
+dcnum/feat/feat_background/bg_sparse_median.py,sha256=87tfB6MoNeTmv9vPBAbfkzNW1m7ESQOg_-rGvm8kY_k,22071
 dcnum/feat/feat_brightness/__init__.py,sha256=o6AebVlmydwNgVF5kW6ITqJyFreoKrU3Ki_3EC8If-s,155
 dcnum/feat/feat_brightness/bright_all.py,sha256=vf8xaYBdKD24hHUXdkI0_S7nbr7m49KW6gvuWvbHDVg,4545
 dcnum/feat/feat_brightness/common.py,sha256=JX49EszYDmnvoOKXFVV1CalEIWRmOuY5EryNbqGbdac,156
@@ -20,17 +20,17 @@ dcnum/feat/feat_texture/__init__.py,sha256=6StM9S540UVtdFFR3bHa7nfCTomeVdoo7Uy9C
 dcnum/feat/feat_texture/common.py,sha256=COXHpXS-7DMouGu3WF83I76L02Sr7P9re4lxajh6g0E,439
 dcnum/feat/feat_texture/tex_all.py,sha256=_5H3sXYRN0Uq2eUHn3XUyEHkU_tncEqbqJTC-HZcnGY,5198
 dcnum/logic/__init__.py,sha256=7J3GrwJInNQbrLk61HRIV7X7p69TAIbMYpR34hh6u14,177
-dcnum/logic/ctrl.py,sha256=sgn1gxctsiPjzD4hFQpzcPGeHP08yjMhWuwG0ocijeY,36182
+dcnum/logic/ctrl.py,sha256=3ovEkaxy4ASpXvLsZMO0pSueqZfQgLkoy3fnH2Gweq8,38791
 dcnum/logic/job.py,sha256=9BN2WjYqjjJuLnfNZAtQ2Nn47Glo2jVrivDodGJoqlQ,7713
 dcnum/logic/json_encoder.py,sha256=cxMnqisbKEVf-rVcw6rK2BBAb6iz_hKFaGl81kK36lQ,571
 dcnum/meta/__init__.py,sha256=AVqRgyKXO1orKnE305h88IBvoZ1oz6X11HN1WP5nGvg,60
 dcnum/meta/paths.py,sha256=J_ikeHzd7gEeRgAKjuayz3x6q4h1fOiDadM-ZxhAGm4,1053
-dcnum/meta/ppid.py,sha256=OD79NrZ8waC3julwdH8NjneUuXqSRSHsUDpKzT5pdyU,8432
+dcnum/meta/ppid.py,sha256=X46w3hoBW0zxKY1avokC2Wex6E3aqV4LaMmpbCFYrXg,8432
 dcnum/read/__init__.py,sha256=ksLdV8EkOU3EPje8teCOSehcUeGAZfg9TQ5ltuEUgls,216
 dcnum/read/cache.py,sha256=lisrGG7AyvVitf0h92wh5FvYCsxa0pWyGcAyYwGP-LQ,6471
 dcnum/read/const.py,sha256=GG9iyXDtEldvJYOBnhZjlimzIeBMAt4bSr2-xn2gzzc,464
-dcnum/read/hdf5_data.py,sha256=g1kcq9nK2eA063qlAWyvYq1UuRCxdCmWqjK8VTxE8Hk,23603
-dcnum/read/mapped.py,sha256=UryArlrIsHxjOyimBL2Nooi3r73zuGtnGdqdxa6PK_g,3076
+dcnum/read/hdf5_data.py,sha256=JVk9YWw1rPgTPxaMZsw2ehk4FJq9UqhmB1SW7yhPw50,25867
+dcnum/read/mapped.py,sha256=zU2fYdZfLNHn0rKHxDzBhNFMu4--WWa8nSeE2likyZA,3637
 dcnum/segm/__init__.py,sha256=9cLEAd3JWE8IGqDHV-eSDIYOGBfOepd8OcebtNs8Omk,309
 dcnum/segm/segm_thresh.py,sha256=iVhvIhzO0Gw0t3rXOgH71rOI0CNjJJQq4Gg6BulUhK8,948
 dcnum/segm/segmenter.py,sha256=FWLFDBR-x_85ku2rObA2F-QBrM4IUaUL-YHChLagVvM,14902
@@ -44,12 +44,12 @@ dcnum/segm/segm_torch/segm_torch_sto.py,sha256=PTOJrP_FkaxZZul8lM4VA2HL3KyxrheDD
 dcnum/segm/segm_torch/torch_model.py,sha256=5aL6SwSvg1N2gATEGBhP3aA4WTHlvGzQVYuizmh0LrU,3187
 dcnum/segm/segm_torch/torch_postproc.py,sha256=ctirQTmsZnuZGIxkwFWN9arRneHRYJUxaJ_ZyCgjByM,3311
 dcnum/segm/segm_torch/torch_preproc.py,sha256=kjabu76paw23kO7RP7Ik6IY60Kk1VBAHKBAedflA0aQ,4002
-dcnum/write/__init__.py,sha256=QvWHeZmjHI18i-YlGYuzN3i7dVWY9UCReKchrJ-gif0,260
+dcnum/write/__init__.py,sha256=sK79IlvCFIqf2oFABVeyYedMnHOsEIQpxAauEeNO-Tw,273
 dcnum/write/deque_writer_thread.py,sha256=ao7F1yrVKyufgC4rC0Y2_Vt7snuT6KpI7W2qVxcjdhk,1994
 dcnum/write/queue_collector_thread.py,sha256=d_WfdsZdFnFsiAY0zVMwUlA4juIMeiWYmE_-rezBQCE,11734
-dcnum/write/writer.py,sha256=H0XoX6lOi1D1sa-ea5SLVftIQY-gdSIVq4vihYv0ODo,16200
-dcnum-0.23.4.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
-dcnum-0.23.4.dist-info/METADATA,sha256=zq4o1snC1vDJ3l4XdYUAXHGhN0-xIMgQeGUa72BcMys,2280
-dcnum-0.23.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-dcnum-0.23.4.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
-dcnum-0.23.4.dist-info/RECORD,,
+dcnum/write/writer.py,sha256=sFt4O9O4uUabxmBZu776b5pBf_DPDpsz-jLpSsswQ1g,20531
+dcnum-0.24.0.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
+dcnum-0.24.0.dist-info/METADATA,sha256=gvvl8eJLQNtI-p6FUE-AZs1sZ8ZnBwgpT1vVmczgxEs,2280
+dcnum-0.24.0.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
+dcnum-0.24.0.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
+dcnum-0.24.0.dist-info/RECORD,,

{dcnum-0.23.4.dist-info → dcnum-0.24.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (70.1.1)
 Root-Is-Purelib: true
 Tag: py3-none-any