dcnum 0.19.0__tar.gz → 0.20.0__tar.gz

{dcnum-0.19.0 → dcnum-0.20.0}/CHANGELOG

@@ -1,3 +1,13 @@
+0.20.0
+ - feat: support reading mapped basins
+ - feat: support writing mapped-basin-based output files
+ - fix: copy "bg_off" data to output file when copying background data
+ - enh: allow to slice BaseImageChunkCache
+ - enh: sort logs, tables and basins for reproducible access
+ - enh: add more timing information in logs
+ - ref: background progress value is now a double between 0 and 1
+0.19.1
+ - enh: support steps when specifying data slices in `index_mapping`
 0.19.0
  - enh: elevate `HDF5Data`s `index_mapping` to pipeline identifier status
    (this changes the pipeline identifier)

{dcnum-0.19.0/src/dcnum.egg-info → dcnum-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.19.0
+Version: 0.20.0
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.19.0 → dcnum-0.20.0}/src/dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.19.0'
-__version_tuple__ = version_tuple = (0, 19, 0)
+__version__ = version = '0.20.0'
+__version_tuple__ = version_tuple = (0, 20, 0)

{dcnum-0.19.0 → dcnum-0.20.0}/src/dcnum/feat/feat_background/base.py

@@ -62,8 +62,8 @@ class Background(abc.ABC):
 
         #: number of images in the input data
         self.image_count = None
-        #: number of images that have been processed
-        self.image_proc = mp_spawn.Value("L", 0)
+        #: fraction images that have been processed
+        self.image_proc = mp_spawn.Value("d", 0)
 
         #: HDF5Data instance for input data
         self.hdin = None
@@ -185,7 +185,7 @@ class Background(abc.ABC):
         if self.image_count == 0:
             return 0.
         else:
-            return self.image_proc.value / self.image_count
+            return self.image_proc.value
 
     def process(self):
         # Delete any old background data

{dcnum-0.19.0 → dcnum-0.20.0}/src/dcnum/feat/feat_background/bg_copy.py

@@ -9,17 +9,19 @@ class BackgroundCopy(Background):
         pass
 
     def process(self):
-        """Perform median computation on entire input data"""
+        """Copy input data to output dataset"""
         if self.h5in != self.h5out:
-            hin = self.hdin.image_bg.h5ds
-            h5py.h5o.copy(src_loc=hin.parent.id,
-                          src_name=b"image_bg",
-                          dst_loc=self.h5out["events"].id,
-                          dst_name=b"image_bg",
-                          )
+            hin = self.hdin.h5
+            for feat in ["image_bg", "bg_off"]:
+                if feat in hin["events"]:
+                    h5py.h5o.copy(src_loc=hin["events"].id,
+                                  src_name=feat.encode("utf-8"),
+                                  dst_loc=self.h5out["events"].id,
+                                  dst_name=feat.encode("utf-8"),
+                                  )
 
         # set progress to 100%
-        self.image_proc.value = self.image_count
+        self.image_proc.value = 1
 
     def process_approach(self):
         # We do the copying in `process`, because we do not want to modify

{dcnum-0.19.0 → dcnum-0.20.0}/src/dcnum/feat/feat_background/bg_roll_median.py

@@ -184,7 +184,7 @@ class BackgroundRollMed(Background):
                 num_remaining,
                 axis=0)
             self.writer.store_feature_chunk("image_bg", last_chunk)
-            self.image_proc.value += num_remaining
+        self.image_proc.value = 1
 
     def process_next_batch(self):
         """Process one batch of input data"""
@@ -223,7 +223,7 @@ class BackgroundRollMed(Background):
             )
 
         self.current_batch += 1
-        self.image_proc.value += self.batch_size
+        self.image_proc.value += self.batch_size / self.image_count
 
 
 class WorkerRollMed(mp_spawn.Process):

{dcnum-0.19.0 → dcnum-0.20.0}/src/dcnum/feat/feat_background/bg_sparse_median.py

@@ -329,7 +329,7 @@ class BackgroundSparseMed(Background):
             # Fill up remainder of index array with last entry
             bg_idx[idx1:] = ii
 
-        self.image_proc.value = self.image_count
+        self.image_proc.value = 1
 
         # Write background data
         pos = 0
@@ -393,7 +393,7 @@ class BackgroundSparseMed(Background):
 
         self.bg_images[ii] = self.shared_output.reshape(self.image_shape)
 
-        self.image_proc.value = idx_stop
+        self.image_proc.value = idx_stop / self.image_count
 
 
 class WorkerSparseMed(mp_spawn.Process):

{dcnum-0.19.0 → dcnum-0.20.0}/src/dcnum/logic/ctrl.py

@@ -14,6 +14,7 @@ import traceback
 import uuid
 
 import h5py
+import numpy as np
 
 from ..feat.feat_background.base import get_available_background_methods
 from ..feat.queue_event_extractor import QueueEventExtractor
@@ -21,10 +22,10 @@ from ..feat import gate
 from ..feat import EventExtractorManagerThread
 from ..segm import SegmenterManagerThread, get_available_segmenters
 from ..meta import ppid
-from ..read import HDF5Data
-from .._version import version_tuple
+from ..read import HDF5Data, get_mapping_indices
+from .._version import version, version_tuple
 from ..write import (
-    DequeWriterThread, HDF5Writer, QueueCollectorThread,
+    DequeWriterThread, HDF5Writer, QueueCollectorThread, copy_features,
     copy_metadata, create_with_basins, set_default_filter_kwargs
 )
 
@@ -43,6 +44,7 @@ valid_states = [
     "setup",
     "background",
     "segmentation",
+    "plumbing",
     "cleanup",
     "done",
     "error",
@@ -79,8 +81,9 @@ class DCNumJobRunner(threading.Thread):
         # current job state
         self._state = "init"
         # overall progress [0, 1]
-        self._progress_bg = None
-        self._progress_ex = None
+        self._progress_bg = None  # background
+        self._progress_ex = None  # segmentation
+        self._progress_bn = None  # creating basins
         # segmentation frame rate
         self._segm_rate = 0
 
@@ -237,8 +240,12 @@ class DCNumJobRunner(threading.Thread):
         # how much fractional time each processing step takes.
         bgw = 4  # fraction of background
         exw = 27  # fraction of segmentation and feature extraction
+        if self.job["basin_strategy"] == "drain":
+            drw = 15  # because data need to be copied
+        else:
+            drw = 1  # just creating the basins in output file
         clw = 1  # fraction of cleanup operations
-        tot = bgw + exw + clw
+        tot = bgw + exw + drw + clw
         progress = 0
         st = self.state
 
@@ -247,15 +254,22 @@ class DCNumJobRunner(threading.Thread):
             # background already computed
             progress += bgw / tot
         elif self._progress_bg is not None:
-            # This is the image count of the input dataset
-            progress += bgw / tot * (self._progress_bg.value / len(self.draw))
+            # This is the image count of the input dataset.
+            progress += self._progress_bg.value * bgw / tot
 
         # segmentation
         if valid_states.index(st) > valid_states.index("segmentation"):
             # segmentation already done
             progress += exw / tot
         elif self._progress_ex is not None:
-            progress += exw / tot * self._progress_ex
+            progress += self._progress_ex * exw / tot
+
+        # draining basins
+        if valid_states.index(st) > valid_states.index("plumbing"):
+            # plumbing already done
+            progress += drw / tot
+        if self._progress_bn is not None:
+            progress += self._progress_bn * drw / tot
 
         if self.state == "done":
             progress = 1
@@ -371,16 +385,20 @@ class DCNumJobRunner(threading.Thread):
             # Note any new actions that work on `self.path_temp_in` are not
             # reflected in `self.path_temp_out`.
             self.path_temp_in.rename(self.path_temp_out)
-
-        self.state = "cleanup"
-
-        # The user would normally expect the output file to be something
-        # that is self-contained (copying the file wildly across file
-        # systems and network shares should not impair feature availability).
-        # Therefore, we copy any remaining basin-based features to the
-        # temporary output file.
-        if self.job["no_basins_in_output"]:
-            self.task_transfer_basin_data()
+            # Since no segmentation was done, the output file now does not
+            # contain any events. This is not really what we wanted, but we
+            # can still store all features in the output file if required.
+            if self.job["basin_strategy"] == "drain":
+                orig_feats = []
+                for feat in self.draw.h5["events"].keys():
+                    if isinstance(self.draw.h5["events"][feat], h5py.Dataset):
+                        # copy_features does not support Groups
+                        orig_feats.append(feat)
+                with h5py.File(self.path_temp_out, "a") as h5_dst:
+                    copy_features(h5_src=self.draw.h5,
+                                  h5_dst=h5_dst,
+                                  features=orig_feats,
+                                  mapping=None)
 
         with HDF5Writer(self.path_temp_out) as hw:
             # pipeline metadata
@@ -433,7 +451,8 @@ class DCNumJobRunner(threading.Thread):
             with h5py.File(self.job["path_in"]) as h5_src:
                 copy_metadata(h5_src=h5_src,
                               h5_dst=hw.h5,
-                              # don't copy basins
+                              # Don't copy basins, we would have to index-map
+                              # them first.
                               copy_basins=False)
             if redo_seg:
                 # Store the correct measurement identifier. This is used to
@@ -450,6 +469,12 @@ class DCNumJobRunner(threading.Thread):
                 mid_new = f"{mid_cur}_{mid_ap}" if mid_cur else mid_ap
                 hw.h5.attrs["experiment:run identifier"] = mid_new
 
+        # Handle basin data according to the user's request
+        self.state = "plumbing"
+        self.task_enforce_basin_strategy()
+
+        self.state = "cleanup"
+
         trun = datetime.timedelta(seconds=round(time.monotonic() - time_start))
         self.logger.info(f"Run duration: {str(trun)}")
         self.logger.info(time.strftime("Run stop: %Y-%m-%d-%H.%M.%S",
@@ -491,6 +516,115 @@ class DCNumJobRunner(threading.Thread):
             bic.process()
         self.logger.info("Finished background computation")
 
+    def task_enforce_basin_strategy(self):
+        """Transfer basin data from input files to output if requested
+
+        The user specified the "basin_strategy" keyword argument in
+        `self.job`. If this is set to "drain", then copy all basin
+        information from the input file to the output file. If it
+        is set to "tap", then only create basins in the output file.
+        """
+        self._progress_bn = 0
+        t0 = time.perf_counter()
+        # We need to make sure that the features are correctly attributed
+        # from the input files. E.g. if the input file already has
+        # background images, but we recompute the background images, then
+        # we have to use the data from the recomputed background file.
+        # We achieve this by keeping a specific order and only copying those
+        # features that we don't already have in the output file.
+        feats_raw = [
+            # 1. background data from the temporary input image
+            # (this must come before draw [sic!])
+            [self.dtin.h5, ["image_bg", "bg_off"], "critical"],
+            # 2. frame-based scalar features from the raw input file
+            # (e.g. "temp" or "frame")
+            [self.draw.h5, self.draw.features_scalar_frame, "optional"],
+            # 3. image features from the input file
+            [self.draw.h5, ["image", "image_bg", "bg_off"], "optional"],
+        ]
+        with h5py.File(self.path_temp_out, "a") as hout:
+            hw = HDF5Writer(hout)
+            # First, we have to determine the basin mapping from input to
+            # output. This information is stored by the QueueCollectorThread
+            # in the "basinmap0" feature, ready to be used by us.
+            if "index_unmapped" in hout["events"]:
+                # The unmapped indices enumerate the events in the output file
+                # with indices from the mapped input file. E.g. if for the
+                # first image in the input file, two events are found and for
+                # the second image in the input file, three events are found,
+                # then this would contain [0, 0, 1, 1, 1, ...]. If the index
+                # mapping of the input file was set to slice(1, 100), then the
+                # first image would not be there, and we would have
+                # [1, 1, 1, ...].
+                idx_um = hout["events/index_unmapped"]
+
+                # If we want to convert this to an actual basinmap feature,
+                # then we have to convert those indices to indices that map
+                # to the original input HDF5 file.
+                raw_im = self.draw.index_mapping
+                if raw_im is None:
+                    self.logger.info("Input file mapped with basinmap0")
+                    # Create a hard link to save time and space
+                    hout["events/basinmap0"] = hout["events/index_unmapped"]
+                    basinmap = idx_um
+                else:
+                    basinmap = get_mapping_indices(raw_im)[idx_um]
+                    # Store the mapped basin data in the output file.
+                    hw.store_feature_chunk("basinmap0", basinmap)
+                # We don't need them anymore.
+                del hout["events/index_unmapped"]
+
+                # Note that `size_raw != (len(self.draw))` [sic!]. The former
+                # is the size of the raw dataset and the latter is its mapped
+                # size!
+                size_raw = self.draw.h5.attrs["experiment:event count"]
+                if (len(basinmap) == size_raw
+                        and np.all(basinmap == np.arange(size_raw))):
+                    # This means that the images in the input overlap perfectly
+                    # with the images in the output, i.e. a "copy" segmenter
+                    # was used or something is very reproducible.
+                    # We set basinmap to None to be more efficient.
+                    basinmap = None
+
+            else:
+                # The input is identical to the output, because we are using
+                # the same pipeline identifier.
+                basinmap = None
+
+            for hin, feats, importance in feats_raw:
+                # Only consider features that are available in the input
+                # and that are not already in the output.
+                feats = [f for f in feats
+                         if (f in hin["events"] and f not in hout["events"])]
+                if not feats:
+                    continue
+                elif (self.job["basin_strategy"] == "drain"
+                      or importance == "critical"):
+                    # DRAIN: Copy all features over to the output file.
+                    self.logger.debug(f"Transferring {feats} to output file")
+                    copy_features(h5_src=hin,
+                                  h5_dst=hout,
+                                  features=feats,
+                                  mapping=basinmap)
+                else:
+                    # TAP: Create basins for the "optional" features in the
+                    # output file. Note that the "critical" features never
+                    # reach this case.
+                    self.logger.debug(f"Creating basin for {feats}")
+                    # Relative and absolute paths.
+                    pin = pathlib.Path(hin.filename).resolve()
+                    pout = pathlib.Path(hout.filename).resolve()
+                    paths = [pin, os.path.relpath(pin, pout)]
+                    hw.store_basin(name="dcnum basin",
+                                   features=feats,
+                                   mapping=basinmap,
+                                   paths=paths,
+                                   description=f"Created with dcnum {version}",
+                                   )
+                self._progress_bn += 1 / len(feats_raw)
+        t_tot = time.perf_counter() - t0
+        self.logger.info(f"Enforcing basin strategy time: {t_tot:.1f}s")
+
     def task_segment_extract(self):
         self.logger.info("Starting segmentation and feature extraction")
         # Start writer thread
@@ -629,21 +763,6 @@ class DCNumJobRunner(threading.Thread):
 
         self.logger.info("Finished segmentation and feature extraction")
 
-    def task_transfer_basin_data(self):
-        with h5py.File(self.path_temp_out, "a") as hout:
-            hd = HDF5Data(hout)
-            for ii, _ in enumerate(hd.basins):
-                hindat, features = hd.get_basin_data(ii)
-                for feat in features:
-                    if feat not in hout["events"]:
-                        self.logger.debug(
-                            f"Transferring {feat} to output file")
-                        h5py.h5o.copy(src_loc=hindat.h5["events"].id,
-                                      src_name=feat.encode(),
-                                      dst_loc=hout["events"].id,
-                                      dst_name=feat.encode(),
-                                      )
-
 
 def join_thread_helper(thr, timeout, retries, logger, name):
     for _ in range(retries):

{dcnum-0.19.0 → dcnum-0.20.0}/src/dcnum/logic/job.py

@@ -3,7 +3,8 @@ import copy
 import inspect
 import multiprocessing as mp
 import pathlib
-from typing import Dict
+from typing import Dict, Literal
+import warnings
 
 from ..feat import QueueEventExtractor
 from ..feat.feat_background.base import get_available_background_methods
@@ -27,10 +28,62 @@ class DCNumPipelineJob:
                  feature_kwargs: Dict = None,
                  gate_code: str = "norm",
                  gate_kwargs: Dict = None,
-                 no_basins_in_output: bool = True,
+                 basin_strategy: Literal["drain", "tap"] = "drain",
+                 no_basins_in_output: bool = None,
                  num_procs: int = None,
                  debug: bool = False,
                  ):
+        """Pipeline job recipe
+
+        Parameters
+        ----------
+        path_in: pathlib.Path | str
+            input data path
+        path_out: pathlib.Path | str
+            output data path
+        data_code: str
+            code of input data reader to use
+        data_kwargs: dict
+            keyword arguments for data reader
+        background_code: str
+            code of background data computer to use
+        background_kwargs: dict
+            keyword arguments for background data computer
+        segmenter_code: str
+            code of segmenter to use
+        segmenter_kwargs: dict
+            keyword arguments for segmenter
+        feature_code: str
+            code of feature extractor
+        feature_kwargs: dict
+            keyword arguments for feature extractor
+        gate_code: str
+            code for gating/event filtering class
+        gate_kwargs: dict
+            keyword arguments for gating/event filtering class
+        basin_strategy: str
+            strategy on how to handle event data; In principle, not all
+            events have to be stored in the output file if basins are
+            defined, linking back to the original file.
+            - You can "drain" all basins which means that the output file
+              will contain all features, but will also be very big.
+            - You can "tap" the basins, including the input file, which means
+              that the output file will be comparatively small.
+        no_basins_in_output: bool
+            Deprecated
+        num_procs: int
+            Number of processes to use
+        debug: bool
+            Whether to be verbose and use threads instead of processes
+        """
+        if no_basins_in_output is not None:
+            warnings.warn("The `no_basins_in_output` keyword argument is "
+                          "deprecated. Please use `basin_strategy` instead.")
+            if no_basins_in_output:
+                basin_strategy = "drain"
+            else:
+                basin_strategy = "tap"
+
         #: initialize keyword arguments for this job
         self.kwargs = {}
         spec = inspect.getfullargspec(DCNumPipelineJob.__init__)

{dcnum-0.19.0 → dcnum-0.20.0}/src/dcnum/read/__init__.py

@@ -2,4 +2,4 @@
 from .cache import md5sum
 from .const import PROTECTED_FEATURES
 from .hdf5_data import HDF5Data, HDF5ImageCache, concatenated_hdf5_data
-from .mapped import get_mapping_indices, get_mapped_object
+from .mapped import get_mapping_indices, get_mapped_object

{dcnum-0.19.0 → dcnum-0.20.0}/src/dcnum/read/cache.py

@@ -22,6 +22,7 @@ class BaseImageChunkCache(abc.ABC):
                  cache_size: int = 2,
                  ):
         self.shape = shape
+        self._dtype = None
         chunk_size = min(shape[0], chunk_size)
         self._len = self.shape[0]
         #: This is a FILO cache for the chunks
@@ -33,12 +34,32 @@ class BaseImageChunkCache(abc.ABC):
         self.num_chunks = int(np.ceil(self._len / (self.chunk_size or 1)))
 
     def __getitem__(self, index):
-        chunk_index, sub_index = self._get_chunk_index_for_index(index)
-        return self.get_chunk(chunk_index)[sub_index]
+        if isinstance(index, (slice, list, np.ndarray)):
+            if isinstance(index, slice):
+                indices = np.arange(index.start or 0,
+                                    index.stop or len(self),
+                                    index.step)
+            else:
+                indices = index
+            array_out = np.empty((len(indices),) + self.image_shape,
+                                 dtype=self.dtype)
+            for ii, idx in enumerate(indices):
+                array_out[ii] = self[idx]
+            return array_out
+        else:
+            chunk_index, sub_index = self._get_chunk_index_for_index(index)
+            return self.get_chunk(chunk_index)[sub_index]
 
     def __len__(self):
         return self._len
 
+    @property
+    def dtype(self):
+        """data type of the image data"""
+        if self._dtype is None:
+            self._dtype = self[0].dtype
+        return self._dtype
+
     @abc.abstractmethod
     def _get_chunk_data(self, chunk_slice):
         """Implemented in subclass to obtain actual data"""
@@ -50,6 +71,7 @@ class BaseImageChunkCache(abc.ABC):
             raise IndexError(
                 f"Index {index} out of bounds for HDF5ImageCache "
                 f"of size {self._len}")
+        index = int(index)  # convert np.uint64 to int, so we get ints below
         chunk_index = index // self.chunk_size
         sub_index = index % self.chunk_size
         return chunk_index, sub_index

{dcnum-0.19.0 → dcnum-0.20.0}/src/dcnum/read/hdf5_data.py

@@ -186,25 +186,27 @@ class HDF5Data:
                     if isinstance(self.meta[key], bytes):
                         self.meta[key] = self.meta[key].decode("utf-8")
                 # logs
-                for key in h5.get("logs", []):
+                for key in sorted(h5.get("logs", {}).keys()):
                     alog = list(h5["logs"][key])
                     if alog:
                         if isinstance(alog[0], bytes):
                             alog = [ll.decode("utf") for ll in alog]
                         self.logs[key] = alog
                 # tables
-                for tab in h5.get("tables", []):
+                for tab in sorted(h5.get("tables", {}).keys()):
                     tabdict = {}
                     for tkey in h5["tables"][tab].dtype.fields.keys():
                         tabdict[tkey] = \
                             np.array(h5["tables"][tab][tkey]).reshape(-1)
                     self.tables[tab] = tabdict
                 # basins
-                for bnkey in h5.get("basins", []):
+                basins = []
+                for bnkey in h5.get("basins", {}).keys():
                     bn_data = "\n".join(
                         [s.decode() for s in h5["basins"][bnkey][:].tolist()])
                     bn_dict = json.loads(bn_data)
-                    self.basins.append(bn_dict)
+                    basins.append(bn_dict)
+                self.basins = sorted(basins, key=lambda x: x["name"])
 
         if state["pixel_size"] is not None:
             self.pixel_size = state["pixel_size"]
@@ -355,8 +357,12 @@ class HDF5Data:
                 elif val == "0":
                     kwargs["index_mapping"] = None
                 elif val.count("-"):
-                    start, stop = [int(v) for v in val.split("-")]
-                    kwargs["index_mapping"] = slice(start, stop)
+                    start, stop, step = val.split("-")
+                    kwargs["index_mapping"] = slice(
+                        None if start == "n" else int(start),
+                        None if stop == "n" else int(stop),
+                        None if step == "n" else int(step)
+                    )
                 else:
                     kwargs["index_mapping"] = int(val)
             else:
@@ -391,7 +397,22 @@ class HDF5Data:
             if path is None:
                 self._basin_data[index] = (None, None)
             else:
-                h5dat = HDF5Data(path, index_mapping=self.index_mapping)
+                feat_basinmap = bn_dict.get("mapping", None)
+                if feat_basinmap is None:
+                    # This is NOT a mapped basin.
+                    index_mapping = self.index_mapping
+                else:
+                    # This is a mapped basin. Create an indexing list.
+                    if self.index_mapping is None:
+                        # The current dataset is not mapped.
+                        basinmap_idx = slice(None)
+                    else:
+                        # The current dataset is also mapped.
+                        basinmap_idx = get_mapping_indices(self.index_mapping)
+                    basinmap = self.h5[f"events/{feat_basinmap}"]
+                    index_mapping = basinmap[basinmap_idx]
+
+                h5dat = HDF5Data(path, index_mapping=index_mapping)
                 features = bn_dict.get("features")
                 if features is None:
                     # Only get the features from the actual HDF5 file.
@@ -416,21 +437,27 @@ class HDF5Data:
         if feat not in self._image_cache:
             if f"events/{feat}" in self.h5:
                 ds = self.h5[f"events/{feat}"]
+                idx_map = self.index_mapping
             else:
+                idx_map = None
                 # search all basins
                 for idx in range(len(self.basins)):
-                    bndat, features = self.get_basin_data(idx)
+                    bn_dat, features = self.get_basin_data(idx)
                     if features is not None:
                         if feat in features:
-                            ds = bndat.h5[f"events/{feat}"]
+                            # HDF5 dataset
+                            ds = bn_dat.h5[f"events/{feat}"]
+                            # Index mapping (taken from the basins which
+                            # already includes the mapping from the current
+                            # instance).
+                            idx_map = bn_dat.index_mapping
                             break
                 else:
                     ds = None
 
             if ds is not None:
                 image = HDF5ImageCache(
-                    h5ds=get_mapped_object(obj=ds,
-                                           index_mapping=self.index_mapping),
+                    h5ds=get_mapped_object(obj=ds, index_mapping=idx_map),
                     cache_size=self.image_cache_size,
                     boolean=feat == "mask")
             else:

{dcnum-0.19.0 → dcnum-0.20.0}/src/dcnum/read/mapped.py

@@ -34,6 +34,16 @@ class MappedHDF5Dataset:
 def get_mapping_indices(
         index_mapping: numbers.Integral | slice | list | np.ndarray
         ):
+    """Return integer numpy array with mapping indices for a range
+
+    Parameters
+    ----------
+    index_mapping: numbers.Integral | slice | list | np.ndarray
+        Several options you have here:
+        - integer: results in np.arrange(integer)
+        - slice: results in np.arrange(slice.start, slice.stop, slice.step)
+        - list or np.ndarray: returns the input as  unit32 array
+    """
     if isinstance(index_mapping, numbers.Integral):
         return _get_mapping_indices_cached(index_mapping)
     elif isinstance(index_mapping, slice):
@@ -54,12 +64,10 @@ def _get_mapping_indices_cached(
         return np.arange(index_mapping)
     elif isinstance(index_mapping, tuple):
         im_slice = slice(*index_mapping)
-        if im_slice.step is not None:
-            raise NotImplementedError("Slices with step not implemented yet")
         if im_slice.stop is None or im_slice.start is None:
             raise NotImplementedError(
                 "Slices must have start and stop defined")
-        return np.arange(im_slice.start, im_slice.stop)
+        return np.arange(im_slice.start, im_slice.stop, im_slice.step)
     elif isinstance(index_mapping, list):
         return np.array(index_mapping, dtype=np.uint32)
     else:

{dcnum-0.19.0 → dcnum-0.20.0}/src/dcnum/write/__init__.py

@@ -2,4 +2,5 @@
 from .deque_writer_thread import DequeWriterThread
 from .queue_collector_thread import EventStash, QueueCollectorThread
 from .writer import (
-    HDF5Writer, copy_metadata, create_with_basins, set_default_filter_kwargs)
+    HDF5Writer, copy_features, copy_metadata, create_with_basins,
+    set_default_filter_kwargs)