dcnum 0.17.0__py3-none-any.whl → 0.23.1__py3-none-any.whl

dcnum/read/cache.py

@@ -1,7 +1,9 @@
+import abc
 import collections
 import functools
 import hashlib
 import pathlib
+from typing import Tuple
 import warnings
 
 import h5py
@@ -13,41 +15,55 @@ class EmptyDatasetWarning(UserWarning):
     pass
 
 
-class HDF5ImageCache:
+class BaseImageChunkCache(abc.ABC):
     def __init__(self,
-                 h5ds: h5py.Dataset,
+                 shape: Tuple[int],
                  chunk_size: int = 1000,
                  cache_size: int = 2,
-                 boolean: bool = False):
-        """An HDF5 image cache
-
-        Deformability cytometry data files commonly contain image stacks
-        that are chunked in various ways. Loading just a single image
-        can be time-consuming, because an entire HDF5 chunk has to be
-        loaded, decompressed and from that one image extracted. The
-        `HDF5ImageCache` class caches the chunks from the HDF5 files
-        into memory, making single-image-access very fast.
-        """
-        self.shape = h5ds.shape
+                 ):
+        self.shape = shape
+        self._dtype = None
+        chunk_size = min(shape[0], chunk_size)
         self._len = self.shape[0]
-        if self._len == 0:
-            warnings.warn(f"Input image '{h5ds.name}' in "
-                          f"file {h5ds.file.filename} has zero length",
-                          EmptyDatasetWarning)
-        # TODO:
-        # - adjust chunking to multiples of the chunks in the dataset
-        #   (which might slightly speed up things)
-        chunk_size = min(h5ds.shape[0], chunk_size)
-        self.h5ds = h5ds
-        self.chunk_size = chunk_size
-        self.boolean = boolean
-        self.cache_size = cache_size
         #: This is a FILO cache for the chunks
         self.cache = collections.OrderedDict()
         self.image_shape = self.shape[1:]
         self.chunk_shape = (chunk_size,) + self.shape[1:]
+        self.chunk_size = chunk_size
+        self.cache_size = cache_size
         self.num_chunks = int(np.ceil(self._len / (self.chunk_size or 1)))
 
+    def __getitem__(self, index):
+        if isinstance(index, (slice, list, np.ndarray)):
+            if isinstance(index, slice):
+                indices = np.arange(index.start or 0,
+                                    index.stop or len(self),
+                                    index.step)
+            else:
+                indices = index
+            array_out = np.empty((len(indices),) + self.image_shape,
+                                 dtype=self.dtype)
+            for ii, idx in enumerate(indices):
+                array_out[ii] = self[idx]
+            return array_out
+        else:
+            chunk_index, sub_index = self._get_chunk_index_for_index(index)
+            return self.get_chunk(chunk_index)[sub_index]
+
+    def __len__(self):
+        return self._len
+
+    @property
+    def dtype(self):
+        """data type of the image data"""
+        if self._dtype is None:
+            self._dtype = self[0].dtype
+        return self._dtype
+
+    @abc.abstractmethod
+    def _get_chunk_data(self, chunk_slice):
+        """Implemented in subclass to obtain actual data"""
+
     def _get_chunk_index_for_index(self, index):
         if index < 0:
             index = self._len + index
@@ -55,30 +71,19 @@ class HDF5ImageCache:
             raise IndexError(
                 f"Index {index} out of bounds for HDF5ImageCache "
                 f"of size {self._len}")
+        index = int(index)  # convert np.uint64 to int, so we get ints below
         chunk_index = index // self.chunk_size
         sub_index = index % self.chunk_size
         return chunk_index, sub_index
 
-    def __getitem__(self, index):
-        chunk_index, sub_index = self._get_chunk_index_for_index(index)
-        return self.get_chunk(chunk_index)[sub_index]
-
-    def __len__(self):
-        return self._len
-
     def get_chunk(self, chunk_index):
         """Return one chunk of images"""
         if chunk_index not in self.cache:
-            fslice = slice(self.chunk_size * chunk_index,
-                           self.chunk_size * (chunk_index + 1)
-                           )
-            data = self.h5ds[fslice]
-            if self.boolean:
-                data = np.array(data, dtype=bool)
-            self.cache[chunk_index] = data
-            if len(self.cache) > self.cache_size:
+            if len(self.cache) >= self.cache_size:
                 # Remove the first item
                 self.cache.popitem(last=False)
+            data = self._get_chunk_data(self.get_chunk_slice(chunk_index))
+            self.cache[chunk_index] = data
         return self.cache[chunk_index]
 
     def get_chunk_size(self, chunk_index):
@@ -91,60 +96,77 @@ class HDF5ImageCache:
                 raise IndexError(f"{self} only has {self.num_chunks} chunks!")
             return chunk_size
 
+    def get_chunk_slice(self, chunk_index):
+        """Return the slice corresponding to the chunk index"""
+        ch_slice = slice(self.chunk_size * chunk_index,
+                         self.chunk_size * (chunk_index + 1)
+                         )
+        return ch_slice
+
     def iter_chunks(self):
-        size = self.h5ds.shape[0]
         index = 0
         chunk = 0
         while True:
             yield chunk
             chunk += 1
             index += self.chunk_size
-            if index >= size:
+            if index >= self._len:
                 break
 
 
-class ImageCorrCache:
+class HDF5ImageCache(BaseImageChunkCache):
     def __init__(self,
-                 image: HDF5ImageCache,
-                 image_bg: HDF5ImageCache):
-        self.image = image
-        self.image_bg = image_bg
-        self.chunk_size = image.chunk_size
-        self.num_chunks = image.num_chunks
-        self.h5ds = image.h5ds
-        self.shape = image.shape
-        self.chunk_shape = image.chunk_shape
-        #: This is a FILO cache for the corrected image chunks
-        self.cache = collections.OrderedDict()
-        self.cache_size = image.cache_size
+                 h5ds: h5py.Dataset,
+                 chunk_size: int = 1000,
+                 cache_size: int = 2,
+                 boolean: bool = False):
+        """An HDF5 image cache
 
-    def _get_chunk_index_for_index(self, index):
-        if index < 0:
-            index = len(self.h5ds) + index
-        chunk_index = index // self.chunk_size
-        sub_index = index % self.chunk_size
-        return chunk_index, sub_index
+        Deformability cytometry data files commonly contain image stacks
+        that are chunked in various ways. Loading just a single image
+        can be time-consuming, because an entire HDF5 chunk has to be
+        loaded, decompressed and from that one image extracted. The
+        `HDF5ImageCache` class caches the chunks from the HDF5 files
+        into memory, making single-image-access very fast.
+        """
+        super(HDF5ImageCache, self).__init__(
+            shape=h5ds.shape,
+            chunk_size=chunk_size,
+            cache_size=cache_size)
+        # TODO:
+        # - adjust chunking to multiples of the chunks in the dataset
+        #   (which might slightly speed up things)
+        self.h5ds = h5ds
+        self.boolean = boolean
 
-    def __getitem__(self, index):
-        chunk_index, sub_index = self._get_chunk_index_for_index(index)
-        return self.get_chunk(chunk_index)[sub_index]
+        if self._len == 0:
+            warnings.warn(f"Input image '{h5ds.name}' in "
+                          f"file {h5ds.file.filename} has zero length",
+                          EmptyDatasetWarning)
 
-    def __len__(self):
-        return len(self.image)
+    def _get_chunk_data(self, chunk_slice):
+        data = self.h5ds[chunk_slice]
+        if self.boolean:
+            data = np.array(data, dtype=bool)
+        return data
 
-    def get_chunk(self, chunk_index):
-        if chunk_index not in self.cache:
-            data = np.array(
-                self.image.get_chunk(chunk_index), dtype=np.int16) \
-                - self.image_bg.get_chunk(chunk_index)
-            self.cache[chunk_index] = data
-            if len(self.cache) > self.cache_size:
-                # Remove the first item
-                self.cache.popitem(last=False)
-        return self.cache[chunk_index]
 
-    def iter_chunks(self):
-        return self.image.iter_chunks()
+class ImageCorrCache(BaseImageChunkCache):
+    def __init__(self,
+                 image: HDF5ImageCache,
+                 image_bg: HDF5ImageCache):
+        super(ImageCorrCache, self).__init__(
+            shape=image.shape,
+            chunk_size=image.chunk_size,
+            cache_size=image.cache_size)
+        self.image = image
+        self.image_bg = image_bg
+
+    def _get_chunk_data(self, chunk_slice):
+        data = np.array(
+            self.image._get_chunk_data(chunk_slice), dtype=np.int16) \
+           - self.image_bg._get_chunk_data(chunk_slice)
+        return data
 
 
 @functools.cache

dcnum/read/const.py

@@ -1,15 +1,17 @@
-#: Scalar features that apply to all events in a frame
+#: Scalar features that apply to all events in a frame and which are
+#: not computed for individual events.
 PROTECTED_FEATURES = [
-    "bg_med",
+    "bg_off",
     "flow_rate",
     "frame",
     "g_force",
-    "index_online",
     "pressure",
     "temp",
     "temp_amb",
-    "time"
+    "time",
 ]
 
+# User-defined features may be anything, but if the user needs something
+# very specific for the pipeline, having them protected is a nice feature.
 for ii in range(10):
     PROTECTED_FEATURES.append(f"userdef{ii}")

dcnum/read/hdf5_data.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import hashlib
 import io
 import json
+import numbers
 import pathlib
 import tempfile
 from typing import Dict, BinaryIO, List
@@ -13,6 +15,7 @@ import numpy as np
 
 from .cache import HDF5ImageCache, ImageCorrCache, md5sum
 from .const import PROTECTED_FEATURES
+from .mapped import get_mapped_object, get_mapping_indices
 
 
 class HDF5Data:
@@ -26,12 +29,47 @@ class HDF5Data:
                  logs: Dict[List[str]] = None,
                  tables: Dict[np.ndarray] = None,
                  image_cache_size: int = 2,
+                 index_mapping: int | slice | List | np.ndarray = None,
                  ):
+        """
+
+        Parameters
+        ----------
+        path:
+            path to data file
+        pixel_size:
+            pixel size in µm
+        md5_5m:
+            MD5 sum of the first 5 MiB; computed if not provided
+        meta:
+            metadata dictionary; extracted from HDF5 attributes
+            if not provided
+        basins:
+            list of basin dictionaries; extracted from HDF5 attributes
+            if not provided
+        logs:
+            dictionary of logs; extracted from HDF5 attributes
+            if not provided
+        tables:
+            dictionary of tables; extracted from HDF5 attributes
+            if not provided
+        image_cache_size:
+            size of the image cache to use when accessing image data
+        index_mapping:
+            select only a subset of input events, transparently reducing the
+            size of the dataset, possible data types are
+            - int `N`: use the first `N` events
+            - slice: use the events defined by a slice
+            - list: list of integers specifying the event indices to use
+            Numpy indexing rules apply. E.g. to only process the first
+            100 events, set this to `100` or `slice(0, 100)`.
+        """
         # Init is in __setstate__ so we can pickle this class
         # and use it for multiprocessing.
         if isinstance(path, h5py.File):
             self.h5 = path
             path = path.filename
+
         self.__setstate__({"path": path,
                            "pixel_size": pixel_size,
                            "md5_5m": md5_5m,
@@ -40,6 +78,7 @@ class HDF5Data:
                            "logs": logs,
                            "tables": tables,
                            "image_cache_size": image_cache_size,
+                           "index_mapping": index_mapping,
                            })
 
     def __contains__(self, item):
@@ -53,7 +92,7 @@ class HDF5Data:
 
     def __getitem__(self, feat):
         if feat in ["image", "image_bg", "mask"]:
-            data = self.get_image_cache(feat)
+            data = self.get_image_cache(feat)  # already index-mapped
             if data is None:
                 raise KeyError(f"Feature '{feat}' not found in {self}!")
             else:
@@ -62,19 +101,25 @@ class HDF5Data:
             return self._cache_scalar[feat]
         elif (feat in self.h5["events"]
               and len(self.h5["events"][feat].shape) == 1):  # cache scalar
-            self._cache_scalar[feat] = self.h5["events"][feat][:]
+            if self.index_mapping is None:
+                idx_map = slice(None)  # no mapping indices, just slice
+            else:
+                idx_map = get_mapping_indices(self.index_mapping)
+            self._cache_scalar[feat] = self.h5["events"][feat][idx_map]
             return self._cache_scalar[feat]
         else:
             if feat in self.h5["events"]:
                 # Not cached (possibly slow)
                 warnings.warn(f"Feature {feat} not cached (possibly slow)")
-                return self.h5["events"][feat]
+                return get_mapped_object(
+                    obj=self.h5["events"][feat],
+                    index_mapping=self.index_mapping)
             else:
                 # Check the basins
                 for idx in range(len(self.basins)):
                     bn, bn_features = self.get_basin_data(idx)
                     if bn_features and feat in bn_features:
-                        return bn[feat]
+                        return bn[feat]  # already index-mapped
         # If we got here, then the feature data does not exist.
         raise KeyError(f"Feature '{feat}' not found in {self}!")
 
@@ -86,13 +131,14 @@ class HDF5Data:
                 "logs": self.logs,
                 "tables": self.tables,
                 "basins": self.basins,
-                "image_cache_size": self.image.cache_size
+                "image_cache_size": self.image.cache_size,
+                "index_mapping": self.index_mapping,
                 }
 
     def __setstate__(self, state):
         # Make sure these properties exist (we rely on __init__, because
         # we want this class to be pickable and __init__ is not called by
-        # `pickle.load`.
+        # `pickle.load`).
         # Cached properties
         self._feats = None
         self._keys = None
@@ -116,7 +162,7 @@ class HDF5Data:
         if self.md5_5m is None:
             if isinstance(self.path, pathlib.Path):
                 # 5MB md5sum of input file
-                self.md5_5m = md5sum(self.path, count=80)
+                self.md5_5m = md5sum(self.path, blocksize=65536, count=80)
             else:
                 self.md5_5m = str(uuid.uuid4()).replace("-", "")
         self.meta = state["meta"]
@@ -140,37 +186,44 @@ class HDF5Data:
                     if isinstance(self.meta[key], bytes):
                         self.meta[key] = self.meta[key].decode("utf-8")
                 # logs
-                for key in h5.get("logs", []):
+                for key in sorted(h5.get("logs", {}).keys()):
                     alog = list(h5["logs"][key])
                     if alog:
                         if isinstance(alog[0], bytes):
                             alog = [ll.decode("utf") for ll in alog]
                         self.logs[key] = alog
                 # tables
-                for tab in h5.get("tables", []):
+                for tab in sorted(h5.get("tables", {}).keys()):
                     tabdict = {}
                     for tkey in h5["tables"][tab].dtype.fields.keys():
                         tabdict[tkey] = \
                             np.array(h5["tables"][tab][tkey]).reshape(-1)
                     self.tables[tab] = tabdict
                 # basins
-                for bnkey in h5.get("basins", []):
+                basins = []
+                for bnkey in h5.get("basins", {}).keys():
                     bn_data = "\n".join(
                         [s.decode() for s in h5["basins"][bnkey][:].tolist()])
                     bn_dict = json.loads(bn_data)
-                    self.basins.append(bn_dict)
+                    basins.append(bn_dict)
+                self.basins = sorted(basins, key=lambda x: x["name"])
 
         if state["pixel_size"] is not None:
             self.pixel_size = state["pixel_size"]
 
         self.image_cache_size = state["image_cache_size"]
 
+        self.index_mapping = state["index_mapping"]
+
         if self.h5 is None:
             self.h5 = h5py.File(self.path, libver="latest")
 
     def __len__(self):
         if self._len is None:
-            self._len = self.h5.attrs["experiment:event count"]
+            if self.index_mapping is not None:
+                self._len = get_mapping_indices(self.index_mapping).size
+            else:
+                self._len = self.h5.attrs["experiment:event count"]
         return self._len
 
     @property
@@ -244,7 +297,9 @@ class HDF5Data:
         self.h5.close()
 
     def get_ppid(self):
-        return self.get_ppid_from_ppkw({"pixel_size": self.pixel_size})
+        return self.get_ppid_from_ppkw(
+            {"pixel_size": self.pixel_size,
+             "index_mapping": self.index_mapping})
 
     @classmethod
     def get_ppid_code(cls):
@@ -255,20 +310,64 @@ class HDF5Data:
         # Data does not really fit into the PPID scheme we use for the rest
         # of the pipeline. This implementation here is custom.
         code = cls.get_ppid_code()
-        kwid = f"p={kwargs['pixel_size']:.8f}".rstrip("0")
+        # pixel size
+        ppid_ps = f"{kwargs['pixel_size']:.8f}".rstrip("0")
+        # index mapping
+        ppid_im = cls.get_ppid_index_mapping(kwargs.get("index_mapping", None))
+        kwid = "^".join([f"p={ppid_ps}", f"i={ppid_im}"])
         return ":".join([code, kwid])
 
+    @staticmethod
+    def get_ppid_index_mapping(index_mapping):
+        """Return the pipeline identifier part for index mapping"""
+        im = index_mapping
+        if im is None:
+            dim = "0"
+        elif isinstance(im, numbers.Integral):
+            dim = f"{im}"
+        elif isinstance(im, slice):
+            dim = (f"{im.start if im.start is not None else 'n'}"
+                   + f"-{im.stop if im.stop is not None else 'n'}"
+                   + f"-{im.step if im.step is not None else 'n'}"
+                   )
+        elif isinstance(im, (list, np.ndarray)):
+            idhash = hashlib.md5(
+                np.array(im, dtype=np.uint32).tobytes()).hexdigest()
+            dim = f"h-{idhash[:8]}"
+        else:
+            dim = "unknown"
+        return dim
+
     @staticmethod
     def get_ppkw_from_ppid(dat_ppid):
         # Data does not fit in the PPID scheme we use, but we still
         # would like to pass pixel_size to __init__ if we need it.
-        code, pp_dat_kwargs = dat_ppid.split(":")
+        code, kwargs_str = dat_ppid.split(":")
         if code != HDF5Data.get_ppid_code():
             raise ValueError(f"Could not find data method '{code}'!")
-        p, val = pp_dat_kwargs.split("=")
-        if p != "p":
-            raise ValueError(f"Invalid parameter '{p}'!")
-        return {"pixel_size": float(val)}
+        kwitems = kwargs_str.split("^")
+        kwargs = {}
+        for item in kwitems:
+            var, val = item.split("=")
+            if var == "p":
+                kwargs["pixel_size"] = float(val)
+            elif var == "i":
+                if val.startswith("h-") or val == "unknown":
+                    raise ValueError(f"Cannot invert index mapping {val}")
+                elif val == "0":
+                    kwargs["index_mapping"] = None
+                elif val.count("-"):
+                    start, stop, step = val.split("-")
+                    kwargs["index_mapping"] = slice(
+                        None if start == "n" else int(start),
+                        None if stop == "n" else int(stop),
+                        None if step == "n" else int(step)
+                    )
+                else:
+                    kwargs["index_mapping"] = int(val)
+            else:
+                raise ValueError(f"Invalid parameter '{var}'!")
+        return kwargs
 
     def get_basin_data(self, index):
         """Return HDF5Data info for a basin index in `self.basins`
@@ -298,7 +397,22 @@ class HDF5Data:
             if path is None:
                 self._basin_data[index] = (None, None)
             else:
-                h5dat = HDF5Data(path)
+                feat_basinmap = bn_dict.get("mapping", None)
+                if feat_basinmap is None:
+                    # This is NOT a mapped basin.
+                    index_mapping = self.index_mapping
+                else:
+                    # This is a mapped basin. Create an indexing list.
+                    if self.index_mapping is None:
+                        # The current dataset is not mapped.
+                        basinmap_idx = slice(None)
+                    else:
+                        # The current dataset is also mapped.
+                        basinmap_idx = get_mapping_indices(self.index_mapping)
+                    basinmap = self.h5[f"events/{feat_basinmap}"]
+                    index_mapping = basinmap[basinmap_idx]
+
+                h5dat = HDF5Data(path, index_mapping=index_mapping)
                 features = bn_dict.get("features")
                 if features is None:
                     # Only get the features from the actual HDF5 file.
@@ -323,20 +437,27 @@ class HDF5Data:
         if feat not in self._image_cache:
             if f"events/{feat}" in self.h5:
                 ds = self.h5[f"events/{feat}"]
+                idx_map = self.index_mapping
             else:
+                idx_map = None
                 # search all basins
                 for idx in range(len(self.basins)):
-                    bndat, features = self.get_basin_data(idx)
+                    bn_dat, features = self.get_basin_data(idx)
                     if features is not None:
                         if feat in features:
-                            ds = bndat.h5[f"events/{feat}"]
+                            # HDF5 dataset
+                            ds = bn_dat.h5[f"events/{feat}"]
+                            # Index mapping (taken from the basins which
+                            # already includes the mapping from the current
+                            # instance).
+                            idx_map = bn_dat.index_mapping
                             break
                 else:
                     ds = None
 
             if ds is not None:
                 image = HDF5ImageCache(
-                    h5ds=ds,
+                    h5ds=get_mapped_object(obj=ds, index_mapping=idx_map),
                     cache_size=self.image_cache_size,
                     boolean=feat == "mask")
             else:
@@ -386,6 +507,7 @@ def concatenated_hdf5_data(paths: List[pathlib.Path],
     - If one of the input files does not contain a feature from the first
       input `paths`, then a `ValueError` is raised. Use the `features`
       argument to specify which features you need instead.
+    - Basins are not considered.
     """
     h5kwargs = {"mode": "w", "libver": "latest"}
     if isinstance(path_out, (pathlib.Path, str)):
@@ -432,6 +554,7 @@ def concatenated_hdf5_data(paths: List[pathlib.Path],
                     if not isinstance(h5["events"][feat], h5py.Dataset):
                         warnings.warn(
                             f"Ignoring {feat}; not implemented yet!")
+                        continue
                     if feat in ["frame", "time"]:
                         continue
                     shapes.setdefault(feat, []).append(