dcnum 0.13.2__py3-none-any.whl → 0.23.1__py3-none-any.whl

dcnum/read/hdf5_data.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import hashlib
 import io
 import json
+import numbers
 import pathlib
 import tempfile
 from typing import Dict, BinaryIO, List
@@ -13,6 +15,7 @@ import numpy as np
 
 from .cache import HDF5ImageCache, ImageCorrCache, md5sum
 from .const import PROTECTED_FEATURES
+from .mapped import get_mapped_object, get_mapping_indices
 
 
 class HDF5Data:
@@ -25,13 +28,48 @@ class HDF5Data:
                  basins: List[Dict[List[str] | str]] = None,
                  logs: Dict[List[str]] = None,
                  tables: Dict[np.ndarray] = None,
-                 image_cache_size: int = 5,
+                 image_cache_size: int = 2,
+                 index_mapping: int | slice | List | np.ndarray = None,
                  ):
+        """
+
+        Parameters
+        ----------
+        path:
+            path to data file
+        pixel_size:
+            pixel size in µm
+        md5_5m:
+            MD5 sum of the first 5 MiB; computed if not provided
+        meta:
+            metadata dictionary; extracted from HDF5 attributes
+            if not provided
+        basins:
+            list of basin dictionaries; extracted from HDF5 attributes
+            if not provided
+        logs:
+            dictionary of logs; extracted from HDF5 attributes
+            if not provided
+        tables:
+            dictionary of tables; extracted from HDF5 attributes
+            if not provided
+        image_cache_size:
+            size of the image cache to use when accessing image data
+        index_mapping:
+            select only a subset of input events, transparently reducing the
+            size of the dataset, possible data types are
+            - int `N`: use the first `N` events
+            - slice: use the events defined by a slice
+            - list: list of integers specifying the event indices to use
+            Numpy indexing rules apply. E.g. to only process the first
+            100 events, set this to `100` or `slice(0, 100)`.
+        """
         # Init is in __setstate__ so we can pickle this class
         # and use it for multiprocessing.
         if isinstance(path, h5py.File):
             self.h5 = path
             path = path.filename
+
         self.__setstate__({"path": path,
                            "pixel_size": pixel_size,
                            "md5_5m": md5_5m,
@@ -40,6 +78,7 @@ class HDF5Data:
                            "logs": logs,
                            "tables": tables,
                            "image_cache_size": image_cache_size,
+                           "index_mapping": index_mapping,
                            })
 
     def __contains__(self, item):
@@ -53,24 +92,34 @@ class HDF5Data:
 
     def __getitem__(self, feat):
         if feat in ["image", "image_bg", "mask"]:
-            return self.get_image_cache(feat)
+            data = self.get_image_cache(feat)  # already index-mapped
+            if data is None:
+                raise KeyError(f"Feature '{feat}' not found in {self}!")
+            else:
+                return data
         elif feat in self._cache_scalar:  # check for scalar cached
             return self._cache_scalar[feat]
         elif (feat in self.h5["events"]
               and len(self.h5["events"][feat].shape) == 1):  # cache scalar
-            self._cache_scalar[feat] = self.h5["events"][feat][:]
+            if self.index_mapping is None:
+                idx_map = slice(None)  # no mapping indices, just slice
+            else:
+                idx_map = get_mapping_indices(self.index_mapping)
+            self._cache_scalar[feat] = self.h5["events"][feat][idx_map]
             return self._cache_scalar[feat]
         else:
             if feat in self.h5["events"]:
                 # Not cached (possibly slow)
                 warnings.warn(f"Feature {feat} not cached (possibly slow)")
-                return self.h5["events"][feat]
+                return get_mapped_object(
+                    obj=self.h5["events"][feat],
+                    index_mapping=self.index_mapping)
             else:
                 # Check the basins
                 for idx in range(len(self.basins)):
                     bn, bn_features = self.get_basin_data(idx)
                     if bn_features and feat in bn_features:
-                        return bn[feat]
+                        return bn[feat]  # already index-mapped
         # If we got here, then the feature data does not exist.
         raise KeyError(f"Feature '{feat}' not found in {self}!")
 
@@ -82,13 +131,14 @@ class HDF5Data:
                 "logs": self.logs,
                 "tables": self.tables,
                 "basins": self.basins,
-                "image_cache_size": self.image.cache_size
+                "image_cache_size": self.image.cache_size,
+                "index_mapping": self.index_mapping,
                 }
 
     def __setstate__(self, state):
         # Make sure these properties exist (we rely on __init__, because
         # we want this class to be pickable and __init__ is not called by
-        # `pickle.load`.
+        # `pickle.load`).
         # Cached properties
         self._feats = None
         self._keys = None
@@ -112,7 +162,7 @@ class HDF5Data:
         if self.md5_5m is None:
             if isinstance(self.path, pathlib.Path):
                 # 5MB md5sum of input file
-                self.md5_5m = md5sum(self.path, count=80)
+                self.md5_5m = md5sum(self.path, blocksize=65536, count=80)
             else:
                 self.md5_5m = str(uuid.uuid4()).replace("-", "")
         self.meta = state["meta"]
@@ -136,46 +186,44 @@ class HDF5Data:
                     if isinstance(self.meta[key], bytes):
                         self.meta[key] = self.meta[key].decode("utf-8")
                 # logs
-                for key in h5.get("logs", []):
+                for key in sorted(h5.get("logs", {}).keys()):
                     alog = list(h5["logs"][key])
                     if alog:
                         if isinstance(alog[0], bytes):
                             alog = [ll.decode("utf") for ll in alog]
                         self.logs[key] = alog
                 # tables
-                for tab in h5.get("tables", []):
+                for tab in sorted(h5.get("tables", {}).keys()):
                     tabdict = {}
                     for tkey in h5["tables"][tab].dtype.fields.keys():
                         tabdict[tkey] = \
                             np.array(h5["tables"][tab][tkey]).reshape(-1)
                     self.tables[tab] = tabdict
                 # basins
-                for bnkey in h5.get("basins", []):
+                basins = []
+                for bnkey in h5.get("basins", {}).keys():
                     bn_data = "\n".join(
                         [s.decode() for s in h5["basins"][bnkey][:].tolist()])
                     bn_dict = json.loads(bn_data)
-                    self.basins.append(bn_dict)
+                    basins.append(bn_dict)
+                self.basins = sorted(basins, key=lambda x: x["name"])
 
         if state["pixel_size"] is not None:
             self.pixel_size = state["pixel_size"]
-        else:
-            # Set known pixel size if possible
-            did = self.meta.get("setup:identifier", "EMPTY")
-            if (did.startswith("RC-")
-                    and (self.pixel_size < 0.255 or self.pixel_size > 0.275)):
-                warnings.warn(
-                    f"Correcting for invalid pixel size in '{self.path}'!")
-                # Set default pixel size for Rivercyte devices
-                self.pixel_size = 0.2645
 
         self.image_cache_size = state["image_cache_size"]
 
+        self.index_mapping = state["index_mapping"]
+
         if self.h5 is None:
             self.h5 = h5py.File(self.path, libver="latest")
 
     def __len__(self):
         if self._len is None:
-            self._len = self.h5.attrs["experiment:event count"]
+            if self.index_mapping is not None:
+                self._len = get_mapping_indices(self.index_mapping).size
+            else:
+                self._len = self.h5.attrs["experiment:event count"]
         return self._len
 
     @property
@@ -217,7 +265,10 @@ class HDF5Data:
         return self.meta.get("imaging:pixel size", 0)
 
     @pixel_size.setter
-    def pixel_size(self, pixel_size):
+    def pixel_size(self, pixel_size: float):
+        # Reduce pixel_size accuracy to 8 digits after the point to
+        # enforce pipeline reproducibility (see get_ppid_from_ppkw).
+        pixel_size = float(f"{pixel_size:.8f}")
         self.meta["imaging:pixel size"] = pixel_size
 
     @property
@@ -230,7 +281,7 @@ class HDF5Data:
         """
         if self._feats is None:
             feats = []
-            for feat in self.h5["events"]:
+            for feat in self.keys():
                 if feat in PROTECTED_FEATURES:
                     feats.append(feat)
             self._feats = feats
@@ -245,6 +296,79 @@ class HDF5Data:
         self._basin_data.clear()
         self.h5.close()
 
+    def get_ppid(self):
+        return self.get_ppid_from_ppkw(
+            {"pixel_size": self.pixel_size,
+             "index_mapping": self.index_mapping})
+
+    @classmethod
+    def get_ppid_code(cls):
+        return "hdf"
+
+    @classmethod
+    def get_ppid_from_ppkw(cls, kwargs):
+        # Data does not really fit into the PPID scheme we use for the rest
+        # of the pipeline. This implementation here is custom.
+        code = cls.get_ppid_code()
+        # pixel size
+        ppid_ps = f"{kwargs['pixel_size']:.8f}".rstrip("0")
+        # index mapping
+        ppid_im = cls.get_ppid_index_mapping(kwargs.get("index_mapping", None))
+        kwid = "^".join([f"p={ppid_ps}", f"i={ppid_im}"])
+        return ":".join([code, kwid])
+
+    @staticmethod
+    def get_ppid_index_mapping(index_mapping):
+        """Return the pipeline identifier part for index mapping"""
+        im = index_mapping
+        if im is None:
+            dim = "0"
+        elif isinstance(im, numbers.Integral):
+            dim = f"{im}"
+        elif isinstance(im, slice):
+            dim = (f"{im.start if im.start is not None else 'n'}"
+                   + f"-{im.stop if im.stop is not None else 'n'}"
+                   + f"-{im.step if im.step is not None else 'n'}"
+                   )
+        elif isinstance(im, (list, np.ndarray)):
+            idhash = hashlib.md5(
+                np.array(im, dtype=np.uint32).tobytes()).hexdigest()
+            dim = f"h-{idhash[:8]}"
+        else:
+            dim = "unknown"
+        return dim
+
+    @staticmethod
+    def get_ppkw_from_ppid(dat_ppid):
+        # Data does not fit in the PPID scheme we use, but we still
+        # would like to pass pixel_size to __init__ if we need it.
+        code, kwargs_str = dat_ppid.split(":")
+        if code != HDF5Data.get_ppid_code():
+            raise ValueError(f"Could not find data method '{code}'!")
+        kwitems = kwargs_str.split("^")
+        kwargs = {}
+        for item in kwitems:
+            var, val = item.split("=")
+            if var == "p":
+                kwargs["pixel_size"] = float(val)
+            elif var == "i":
+                if val.startswith("h-") or val == "unknown":
+                    raise ValueError(f"Cannot invert index mapping {val}")
+                elif val == "0":
+                    kwargs["index_mapping"] = None
+                elif val.count("-"):
+                    start, stop, step = val.split("-")
+                    kwargs["index_mapping"] = slice(
+                        None if start == "n" else int(start),
+                        None if stop == "n" else int(stop),
+                        None if step == "n" else int(step)
+                    )
+                else:
+                    kwargs["index_mapping"] = int(val)
+            else:
+                raise ValueError(f"Invalid parameter '{var}'!")
+        return kwargs
+
     def get_basin_data(self, index):
         """Return HDF5Data info for a basin index in `self.basins`
 
@@ -273,9 +397,34 @@ class HDF5Data:
             if path is None:
                 self._basin_data[index] = (None, None)
             else:
-                h5dat = HDF5Data(path)
+                feat_basinmap = bn_dict.get("mapping", None)
+                if feat_basinmap is None:
+                    # This is NOT a mapped basin.
+                    index_mapping = self.index_mapping
+                else:
+                    # This is a mapped basin. Create an indexing list.
+                    if self.index_mapping is None:
+                        # The current dataset is not mapped.
+                        basinmap_idx = slice(None)
+                    else:
+                        # The current dataset is also mapped.
+                        basinmap_idx = get_mapping_indices(self.index_mapping)
+                    basinmap = self.h5[f"events/{feat_basinmap}"]
+                    index_mapping = basinmap[basinmap_idx]
+
+                h5dat = HDF5Data(path, index_mapping=index_mapping)
                 features = bn_dict.get("features")
                 if features is None:
+                    # Only get the features from the actual HDF5 file.
+                    # If this file has basins as well, the basin metadata
+                    # should have been copied over to the parent file. This
+                    # makes things a little cleaner, because basins are not
+                    # nested, but all basins are available in the top file.
+                    # See :func:`write.store_metadata` for copying metadata
+                    # between files.
+                    # The writer can still specify "features" in the basin
+                    # metadata, then these basins are indeed nested, and
+                    # we consider that ok as well.
                     features = sorted(h5dat.h5["events"].keys())
                 self._basin_data[index] = (h5dat, features)
         return self._basin_data[index]
@@ -288,20 +437,27 @@ class HDF5Data:
         if feat not in self._image_cache:
             if f"events/{feat}" in self.h5:
                 ds = self.h5[f"events/{feat}"]
+                idx_map = self.index_mapping
             else:
+                idx_map = None
                 # search all basins
                 for idx in range(len(self.basins)):
-                    bndat, features = self.get_basin_data(idx)
+                    bn_dat, features = self.get_basin_data(idx)
                     if features is not None:
                         if feat in features:
-                            ds = bndat.h5[f"events/{feat}"]
+                            # HDF5 dataset
+                            ds = bn_dat.h5[f"events/{feat}"]
+                            # Index mapping (taken from the basins which
+                            # already includes the mapping from the current
+                            # instance).
+                            idx_map = bn_dat.index_mapping
                             break
                 else:
                     ds = None
 
             if ds is not None:
                 image = HDF5ImageCache(
-                    h5ds=ds,
+                    h5ds=get_mapped_object(obj=ds, index_mapping=idx_map),
                     cache_size=self.image_cache_size,
                     boolean=feat == "mask")
             else:
@@ -337,8 +493,8 @@ def concatenated_hdf5_data(paths: List[pathlib.Path],
     path_out:
         If `None`, then the dataset is created in memory. If `True`
         (default), create a file on disk. If a pathlib.Path is specified,
-        the dataset is written to that file. Note that files in memory
-        are liekely not pickable (so don't use for multiprocessing).
+        the dataset is written to that file. Note that datasets in memory
+        are likely not pickable (so don't use them for multiprocessing).
     compute_frame:
         Whether to compute the "events/frame" feature, taking the frame
         data from the input files and properly incrementing them along
@@ -351,6 +507,7 @@ def concatenated_hdf5_data(paths: List[pathlib.Path],
     - If one of the input files does not contain a feature from the first
       input `paths`, then a `ValueError` is raised. Use the `features`
       argument to specify which features you need instead.
+    - Basins are not considered.
     """
     h5kwargs = {"mode": "w", "libver": "latest"}
     if isinstance(path_out, (pathlib.Path, str)):
@@ -386,7 +543,8 @@ def concatenated_hdf5_data(paths: List[pathlib.Path],
                 # get metadata
                 if ii == 0:
                     meta = dict(h5.attrs)
-                    features = featsi
+                    if not features:
+                        features = featsi
                 # make sure number of features are consistent
                 if not set(features) <= set(featsi):
                     raise ValueError(
@@ -396,6 +554,7 @@ def concatenated_hdf5_data(paths: List[pathlib.Path],
                     if not isinstance(h5["events"][feat], h5py.Dataset):
                         warnings.warn(
                             f"Ignoring {feat}; not implemented yet!")
+                        continue
                     if feat in ["frame", "time"]:
                         continue
                     shapes.setdefault(feat, []).append(

dcnum/read/mapped.py

@@ -0,0 +1,87 @@
+import functools
+
+import numbers
+
+import h5py
+import numpy as np
+
+
+class MappedHDF5Dataset:
+    def __init__(self,
+                 h5ds: h5py.Dataset,
+                 mapping_indices: np.ndarray):
+        """An index-mapped object for accessing an HDF5 dataset
+
+        Parameters
+        ----------
+        h5ds: h5py.Dataset
+            HDF5 dataset from which to map data
+        mapping_indices: np.ndarray
+            numpy indexing array containing integer indices
+        """
+        self.h5ds = h5ds
+        self.mapping_indices = mapping_indices
+        self.shape = (mapping_indices.size,) + h5ds.shape[1:]
+
+    def __getitem__(self, idx):
+        if isinstance(idx, numbers.Integral):
+            return self.h5ds[self.mapping_indices[idx]]
+        else:
+            idx_mapped = self.mapping_indices[idx]
+            return self.h5ds[idx_mapped]
+
+
+def get_mapping_indices(
+        index_mapping: numbers.Integral | slice | list | np.ndarray
+        ):
+    """Return integer numpy array with mapping indices for a range
+
+    Parameters
+    ----------
+    index_mapping: numbers.Integral | slice | list | np.ndarray
+        Several options you have here:
+        - integer: results in np.arrange(integer)
+        - slice: results in np.arrange(slice.start, slice.stop, slice.step)
+        - list or np.ndarray: returns the input as  unit32 array
+    """
+    if isinstance(index_mapping, numbers.Integral):
+        return _get_mapping_indices_cached(index_mapping)
+    elif isinstance(index_mapping, slice):
+        return _get_mapping_indices_cached(
+            (index_mapping.start, index_mapping.stop, index_mapping.step))
+    elif isinstance(index_mapping, (np.ndarray, list)):
+        return np.array(index_mapping, dtype=np.uint32)
+    else:
+        raise ValueError(f"Invalid type for `index_mapping`: "
+                         f"{type(index_mapping)} ({index_mapping})")
+
+
+@functools.lru_cache(maxsize=100)
+def _get_mapping_indices_cached(
+        index_mapping: numbers.Integral | tuple
+        ):
+    if isinstance(index_mapping, numbers.Integral):
+        return np.arange(index_mapping)
+    elif isinstance(index_mapping, tuple):
+        im_slice = slice(*index_mapping)
+        if im_slice.stop is None or im_slice.start is None:
+            raise NotImplementedError(
+                "Slices must have start and stop defined")
+        return np.arange(im_slice.start, im_slice.stop, im_slice.step)
+    elif isinstance(index_mapping, list):
+        return np.array(index_mapping, dtype=np.uint32)
+    else:
+        raise ValueError(f"Invalid type for cached `index_mapping`: "
+                         f"{type(index_mapping)} ({index_mapping})")
+
+
+def get_mapped_object(obj, index_mapping=None):
+    if index_mapping is None:
+        return obj
+    elif isinstance(obj, h5py.Dataset):
+        return MappedHDF5Dataset(
+            obj,
+            mapping_indices=get_mapping_indices(index_mapping))
+    else:
+        raise ValueError(f"No recipe to convert object of type {type(obj)} "
+                         f"({obj}) to an index-mapped object")

dcnum/segm/__init__.py

@@ -1,18 +1,9 @@
 # flake8: noqa: F401
-import functools
-
-from .segmenter import Segmenter
-from .segmenter_cpu import CPUSegmenter
-from .segmenter_gpu import GPUSegmenter
+from .segmenter import (
+    Segmenter, SegmenterNotApplicableError, get_available_segmenters
+)
+from .segmenter_mpo import MPOSegmenter
+from .segmenter_sto import STOSegmenter
 from .segmenter_manager_thread import SegmenterManagerThread
 from . import segm_thresh
-
-
-@functools.cache
-def get_available_segmenters():
-    """Return dictionary of available segmenters"""
-    segmenters = {}
-    for scls in Segmenter.__subclasses__():
-        for cls in scls.__subclasses__():
-            segmenters[cls.key()] = cls
-    return segmenters
+from . import segm_torch

dcnum/segm/segm_thresh.py

@@ -1,26 +1,19 @@
-from .segmenter_cpu import CPUSegmenter
+from .segmenter_mpo import MPOSegmenter
 
 
-class SegmentThresh(CPUSegmenter):
+class SegmentThresh(MPOSegmenter):
     mask_postprocessing = True
     mask_default_kwargs = {
         "clear_border": True,
         "fill_holes": True,
         "closing_disk": 2,
     }
-
-    def __init__(self, thresh=-6, *args, **kwargs):
-        """Simple image thresholding segmentation
-
-        Parameters
-        ----------
-        """
-        super(SegmentThresh, self).__init__(thresh=thresh, *args, **kwargs)
+    requires_background_correction = True
 
     @staticmethod
-    def segment_approach(image, *,
-                         thresh: float = -6):
-        """Mask retrieval as it is done in Shape-In
+    def segment_algorithm(image, *,
+                          thresh: float = -6):
+        """Mask retrieval using basic thresholding
 
         Parameters
         ----------
@@ -34,7 +27,7 @@ class SegmentThresh(CPUSegmenter):
         Returns
         -------
         mask: 2d boolean ndarray
-            Mask image for the give index
+            Mask image for the given index
         """
         assert thresh < 0, "threshold values above zero not supported!"
         return image < thresh

dcnum/segm/segm_torch/__init__.py

@@ -0,0 +1,19 @@
+import importlib
+
+try:
+    torch = importlib.import_module("torch")
+    req_maj = 2
+    req_min = 3
+    ver_tuple = torch.__version__.split(".")
+    act_maj = int(ver_tuple[0])
+    act_min = int(ver_tuple[1])
+    if act_maj < req_maj or (act_maj == req_maj and act_min < req_min):
+        raise ValueError(f"Your PyTorch version {act_maj}.{act_min} is not "
+                         f"supported, please update to at least "
+                         f"{req_maj}.{req_min}")
+except ImportError:
+    pass
+else:
+    from .segm_torch_mpo import SegmentTorchMPO  # noqa: F401
+    if torch.cuda.is_available():
+        from .segm_torch_sto import SegmentTorchSTO  # noqa: F401