dcnum 0.13.2__py3-none-any.whl → 0.23.1__py3-none-any.whl

dcnum/logic/job.py

@@ -0,0 +1,184 @@
+import collections
+import copy
+import inspect
+import logging
+import multiprocessing as mp
+import pathlib
+from typing import Dict, Literal
+import warnings
+
+from ..feat import QueueEventExtractor
+from ..feat.feat_background.base import get_available_background_methods
+from ..feat.gate import Gate
+from ..meta.ppid import compute_pipeline_hash, DCNUM_PPID_GENERATION
+from ..read import HDF5Data
+from ..segm import get_available_segmenters
+
+
+class DCNumPipelineJob:
+    def __init__(self,
+                 path_in: pathlib.Path | str,
+                 path_out: pathlib.Path | str = None,
+                 data_code: str = "hdf",
+                 data_kwargs: Dict = None,
+                 background_code: str = "sparsemed",
+                 background_kwargs: Dict = None,
+                 segmenter_code: str = "thresh",
+                 segmenter_kwargs: Dict = None,
+                 feature_code: str = "legacy",
+                 feature_kwargs: Dict = None,
+                 gate_code: str = "norm",
+                 gate_kwargs: Dict = None,
+                 basin_strategy: Literal["drain", "tap"] = "drain",
+                 no_basins_in_output: bool = None,
+                 num_procs: int = None,
+                 log_level: int = logging.INFO,
+                 debug: bool = False,
+                 ):
+        """Pipeline job recipe
+
+        Parameters
+        ----------
+        path_in: pathlib.Path | str
+            input data path
+        path_out: pathlib.Path | str
+            output data path
+        data_code: str
+            code of input data reader to use
+        data_kwargs: dict
+            keyword arguments for data reader
+        background_code: str
+            code of background data computer to use
+        background_kwargs: dict
+            keyword arguments for background data computer
+        segmenter_code: str
+            code of segmenter to use
+        segmenter_kwargs: dict
+            keyword arguments for segmenter
+        feature_code: str
+            code of feature extractor
+        feature_kwargs: dict
+            keyword arguments for feature extractor
+        gate_code: str
+            code for gating/event filtering class
+        gate_kwargs: dict
+            keyword arguments for gating/event filtering class
+        basin_strategy: str
+            strategy on how to handle event data; In principle, not all
+            events have to be stored in the output file if basins are
+            defined, linking back to the original file.
+            - You can "drain" all basins which means that the output file
+              will contain all features, but will also be very big.
+            - You can "tap" the basins, including the input file, which means
+              that the output file will be comparatively small.
+        no_basins_in_output: bool
+            Deprecated
+        num_procs: int
+            Number of processes to use
+        log_level: int
+            Logging level to use.
+        debug: bool
+            Whether to set logging level to "DEBUG" and
+            use threads instead of processes
+        """
+        if no_basins_in_output is not None:
+            warnings.warn("The `no_basins_in_output` keyword argument is "
+                          "deprecated. Please use `basin_strategy` instead.")
+            if no_basins_in_output:
+                basin_strategy = "drain"
+            else:
+                basin_strategy = "tap"
+
+        #: initialize keyword arguments for this job
+        self.kwargs = {}
+        spec = inspect.getfullargspec(DCNumPipelineJob.__init__)
+        locs = locals()
+        for arg in spec.args:
+            if arg == "self":
+                continue
+            value = locs[arg]
+            if value is None and spec.annotations[arg] is Dict:
+                value = {}
+            self.kwargs[arg] = value
+        # Set default pixel size for this job
+        if "pixel_size" not in self.kwargs["data_kwargs"]:
+            # Extract from input file
+            with HDF5Data(path_in) as hd:
+                self.kwargs["data_kwargs"]["pixel_size"] = hd.pixel_size
+        # Set default output path
+        if path_out is None:
+            pin = pathlib.Path(path_in)
+            path_out = pin.with_name(pin.stem + "_dcn.rtdc")
+        # Set logging level to DEBUG in debugging mode
+        if self.kwargs["debug"]:
+            self.kwargs["log_level"] = logging.DEBUG
+        self.kwargs["path_out"] = pathlib.Path(path_out)
+        # Set default mask kwargs for segmenter
+        self.kwargs["segmenter_kwargs"].setdefault("kwargs_mask", {})
+        # Set default number of processes
+        if num_procs is None:
+            self.kwargs["num_procs"] = mp.cpu_count()
+
+    def __getitem__(self, item):
+        return copy.deepcopy(self.kwargs[item])
+
+    def __getstate__(self):
+        state = copy.deepcopy(self.kwargs)
+        return state
+
+    def __setstate__(self, state):
+        self.kwargs.clear()
+        self.kwargs.update(copy.deepcopy(state))
+
+    def assert_pp_codes(self):
+        """Sanity check of `self.kwargs`"""
+        # PPID classes with only one option
+        for cls, key in [
+            (HDF5Data, "data_code"),
+            (Gate, "gate_code"),
+            (QueueEventExtractor, "feature_code"),
+        ]:
+            code_act = self.kwargs[key]
+            code_exp = cls.get_ppid_code()
+            if code_act != code_exp:
+                raise ValueError(f"Invalid code '{code_act}' for '{key}', "
+                                 f"expected '{code_exp}'!")
+        # PPID classes with multiple options
+        for options, key in [
+            (get_available_background_methods(), "background_code"),
+            (get_available_segmenters(), "segmenter_code"),
+        ]:
+            code_act = self.kwargs[key]
+            if code_act not in options:
+                raise ValueError(f"Invalid code '{code_act}' for '{key}', "
+                                 f"expected one of '{options}'!")
+
+    def get_ppid(self, ret_hash=False, ret_dict=False):
+        self.assert_pp_codes()
+        pp_hash_kw = collections.OrderedDict()
+        pp_hash_kw["gen_id"] = DCNUM_PPID_GENERATION
+        for pp_kw, cls, cls_kw in [
+            ("dat_id", HDF5Data, "data_kwargs"),
+            ("bg_id",
+             get_available_background_methods()[
+                 self.kwargs["background_code"]],
+             "background_kwargs"),
+            ("seg_id",
+             get_available_segmenters()[self.kwargs["segmenter_code"]],
+             "segmenter_kwargs"),
+            ("feat_id", QueueEventExtractor, "feature_kwargs"),
+            ("gate_id", Gate, "gate_kwargs"),
+        ]:
+            pp_hash_kw[pp_kw] = cls.get_ppid_from_ppkw(self.kwargs[cls_kw])
+
+        ppid = "|".join(pp_hash_kw.values())
+
+        ret = [ppid]
+        if ret_hash:
+            pp_hash = compute_pipeline_hash(**pp_hash_kw)
+            ret.append(pp_hash)
+        if ret_dict:
+            ret.append(pp_hash_kw)
+        if len(ret) == 1:
+            ret = ret[0]
+        return ret

dcnum/logic/json_encoder.py

@@ -0,0 +1,19 @@
+import json
+import numbers
+import pathlib
+
+import numpy as np
+
+
+class ExtendedJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, pathlib.Path):
+            return str(obj)
+        elif isinstance(obj, numbers.Integral):
+            return int(obj)
+        elif isinstance(obj, np.bool_):
+            return bool(obj)
+        elif isinstance(obj, slice):
+            return "PYTHON-SLICE", (obj.start, obj.stop, obj.step)
+        # Let the base class default method raise the TypeError
+        return json.JSONEncoder.default(self, obj)

dcnum/meta/__init__.py

@@ -1,2 +1,3 @@
 # flake8: noqa: F401
+from . import paths
 from . import ppid

dcnum/meta/paths.py

@@ -0,0 +1,30 @@
+import pathlib
+
+search_path_registry = {}
+
+
+def register_search_path(topic: str,
+                         search_path: str | pathlib.Path):
+    """Register a search path for a given topic
+
+    Search paths are a global solution for organizing the locations
+    of resources that are part of an analysis pipeline. For instance,
+    if the location of such a file that depends on where your pipeline is
+    running, you can register multiple search paths and the file will
+    be found using :func:`find_file`.
+    """
+    topic_list = search_path_registry.setdefault(topic, [])
+    topic_list.append(pathlib.Path(search_path))
+
+
+def find_file(topic: str,
+              file_name: str):
+    """Find a file in the search path for the given topic"""
+    search_paths = search_path_registry.get(topic, [])
+    for pp in search_paths:
+        pf = pp / file_name
+        if pf.is_file():
+            return pf
+    else:
+        raise KeyError(f"Could not find {file_name} for {topic} in the "
+                       f"registered search paths {search_paths}")

dcnum/meta/ppid.py

@@ -4,17 +4,38 @@ import collections
 import hashlib
 import inspect
 import pathlib
+from typing import Dict, List, Protocol
+import warnings
 
 
 #: Increment this string if there are breaking changes that make
 #: previous pipelines unreproducible.
-DCNUM_PPID_GENERATION = "5"
+DCNUM_PPID_GENERATION = "10"
 
 
-def compute_pipeline_hash(bg_id, seg_id, feat_id, gate_id,
-                          gen_id=DCNUM_PPID_GENERATION):
+class ClassWithPPIDCapabilities(Protocol):
+    def get_ppid(self) -> str:
+        """full pipeline identifier for the class (instance method)"""
+        pass
+
+    def get_ppid_code(self) -> str:
+        """string representing the class in the pipeline (classmethod)"""
+        pass
+
+    def get_ppid_from_ppkw(self) -> str:
+        """pipeline identifier from specific pipeline keywords (classmethod)"""
+        pass
+
+    def get_ppkw_from_ppid(self) -> Dict:
+        """class keywords from full pipeline identifier (staticmethod)"""
+        pass
+
+
+def compute_pipeline_hash(*, bg_id, seg_id, feat_id, gate_id,
+                          dat_id="unknown", gen_id=DCNUM_PPID_GENERATION):
     hasher = hashlib.md5()
-    hasher.update("|".join([gen_id, bg_id, seg_id, feat_id, gate_id]).encode())
+    hasher.update("|".join([
+        gen_id, dat_id, bg_id, seg_id, feat_id, gate_id]).encode())
     pph = hasher.hexdigest()
     return pph
 
@@ -37,7 +58,10 @@ def convert_to_dtype(value, dtype):
     return value
 
 
-def get_class_method_info(class_obj, static_kw_methods=None):
+def get_class_method_info(class_obj: ClassWithPPIDCapabilities,
+                          static_kw_methods: List = None,
+                          static_kw_defaults: Dict = None,
+                          ):
     """Return dictionary of class info with static keyword methods docs
 
     Parameters
@@ -47,10 +71,19 @@ def get_class_method_info(class_obj, static_kw_methods=None):
     static_kw_methods: list of callable
         The methods to inspect; all kwargs-only keyword arguments
         are extracted.
+    static_kw_defaults: dict
+        If a key in this dictionary matches an item in `static_kw_methods`,
+        then these are the default values returned in the "defaults"
+        dictionary. This is used in cases where a base class does
+        implement some annotations, but the subclass does not actually
+        use them, because e.g. they are taken from a property such as is
+        the case for the mask postprocessing of segmenter classes.
     """
+    if static_kw_defaults is None:
+        static_kw_defaults = {}
     doc = class_obj.__doc__ or class_obj.__init__.__doc__
     info = {
-        "key": class_obj.key(),
+        "code": class_obj.get_ppid_code(),
         "doc": doc,
         "title": doc.split("\n")[0],
         }
@@ -60,19 +93,43 @@ def get_class_method_info(class_obj, static_kw_methods=None):
         for mm in static_kw_methods:
             meth = getattr(class_obj, mm)
             spec = inspect.getfullargspec(meth)
-            defau[mm] = spec.kwonlydefaults
+            if mm_defaults := static_kw_defaults.get(mm):
+                defau[mm] = mm_defaults
+            else:
+                defau[mm] = spec.kwonlydefaults or {}
             annot[mm] = spec.annotations
         info["defaults"] = defau
         info["annotations"] = annot
     return info
 
 
-def kwargs_to_ppid(cls, method, kwargs):
-    info = get_class_method_info(cls, [method])
+def kwargs_to_ppid(cls: ClassWithPPIDCapabilities,
+                   method: str,
+                   kwargs: Dict,
+                   allow_invalid_keys: bool = True):
+    info = get_class_method_info(cls, [method, "__init__"])
 
     concat_strings = []
     if info["defaults"][method]:
         kwdefaults = info["defaults"][method]
+        kwdefaults_init = info["defaults"]["__init__"]
+        kw_false = (set(kwargs.keys())
+                    - set(kwdefaults.keys())
+                    - set(kwdefaults_init.keys()))
+        if kw_false:
+            # This should not have happened.
+            msg = (f"Invalid kwargs {kw_false} specified for method "
+                   f"'{method}'! Valid kwargs are"
+                   f"{sorted(kwdefaults.keys())}. If you wrote this "
+                   f"segmenter and had to implement `__init__`, make sure "
+                   f"that it accepts all kwonly-arguments its super class "
+                   f"accepts. If this is not the case, you are probably "
+                   f"passing invalid kwargs to the segmenter."
+                   )
+            if allow_invalid_keys:
+                warnings.warn(msg, UserWarning)
+            else:
+                raise KeyError(msg)
         kwannot = info["annotations"][method]
         kws = list(kwdefaults.keys())
         kws_abrv = get_unique_prefix(kws)

dcnum/read/__init__.py

@@ -2,3 +2,4 @@
 from .cache import md5sum
 from .const import PROTECTED_FEATURES
 from .hdf5_data import HDF5Data, HDF5ImageCache, concatenated_hdf5_data
+from .mapped import get_mapping_indices, get_mapped_object

dcnum/read/cache.py

@@ -1,42 +1,68 @@
+import abc
 import collections
 import functools
 import hashlib
 import pathlib
+from typing import Tuple
+import warnings
 
 import h5py
 import numpy as np
 
 
-class HDF5ImageCache:
+class EmptyDatasetWarning(UserWarning):
+    """Used for files that contain no actual data"""
+    pass
+
+
+class BaseImageChunkCache(abc.ABC):
     def __init__(self,
-                 h5ds: h5py.Dataset,
+                 shape: Tuple[int],
                  chunk_size: int = 1000,
-                 cache_size: int = 5,
-                 boolean: bool = False):
-        """An HDF5 image cache
-
-        Deformability cytometry data files commonly contain image stacks
-        that are chunked in various ways. Loading just a single image
-        can be time-consuming, because an entire HDF5 chunk has to be
-        loaded, decompressed and from that one image extracted. The
-        `HDF5ImageCache` class caches the chunks from the HDF5 files
-        into memory, making single-image-access very fast.
-        """
-        # TODO:
-        # - adjust chunking to multiples of the chunks in the dataset
-        #   (which will slightly speed up things)
-        chunk_size = min(h5ds.shape[0], chunk_size)
-        self.h5ds = h5ds
-        self.chunk_size = chunk_size
-        self.boolean = boolean
-        self.cache_size = cache_size
+                 cache_size: int = 2,
+                 ):
+        self.shape = shape
+        self._dtype = None
+        chunk_size = min(shape[0], chunk_size)
+        self._len = self.shape[0]
         #: This is a FILO cache for the chunks
         self.cache = collections.OrderedDict()
-        self.shape = h5ds.shape
         self.image_shape = self.shape[1:]
         self.chunk_shape = (chunk_size,) + self.shape[1:]
-        self._len = self.shape[0]
-        self.num_chunks = int(np.ceil(self._len / self.chunk_size))
+        self.chunk_size = chunk_size
+        self.cache_size = cache_size
+        self.num_chunks = int(np.ceil(self._len / (self.chunk_size or 1)))
+
+    def __getitem__(self, index):
+        if isinstance(index, (slice, list, np.ndarray)):
+            if isinstance(index, slice):
+                indices = np.arange(index.start or 0,
+                                    index.stop or len(self),
+                                    index.step)
+            else:
+                indices = index
+            array_out = np.empty((len(indices),) + self.image_shape,
+                                 dtype=self.dtype)
+            for ii, idx in enumerate(indices):
+                array_out[ii] = self[idx]
+            return array_out
+        else:
+            chunk_index, sub_index = self._get_chunk_index_for_index(index)
+            return self.get_chunk(chunk_index)[sub_index]
+
+    def __len__(self):
+        return self._len
+
+    @property
+    def dtype(self):
+        """data type of the image data"""
+        if self._dtype is None:
+            self._dtype = self[0].dtype
+        return self._dtype
+
+    @abc.abstractmethod
+    def _get_chunk_data(self, chunk_slice):
+        """Implemented in subclass to obtain actual data"""
 
     def _get_chunk_index_for_index(self, index):
         if index < 0:
@@ -45,30 +71,19 @@ class HDF5ImageCache:
             raise IndexError(
                 f"Index {index} out of bounds for HDF5ImageCache "
                 f"of size {self._len}")
+        index = int(index)  # convert np.uint64 to int, so we get ints below
         chunk_index = index // self.chunk_size
         sub_index = index % self.chunk_size
         return chunk_index, sub_index
 
-    def __getitem__(self, index):
-        chunk_index, sub_index = self._get_chunk_index_for_index(index)
-        return self.get_chunk(chunk_index)[sub_index]
-
-    def __len__(self):
-        return self._len
-
     def get_chunk(self, chunk_index):
         """Return one chunk of images"""
         if chunk_index not in self.cache:
-            fslice = slice(self.chunk_size * chunk_index,
-                           self.chunk_size * (chunk_index + 1)
-                           )
-            data = self.h5ds[fslice]
-            if self.boolean:
-                data = np.array(data, dtype=bool)
-            self.cache[chunk_index] = data
-            if len(self.cache) > self.cache_size:
+            if len(self.cache) >= self.cache_size:
                 # Remove the first item
                 self.cache.popitem(last=False)
+            data = self._get_chunk_data(self.get_chunk_slice(chunk_index))
+            self.cache[chunk_index] = data
         return self.cache[chunk_index]
 
     def get_chunk_size(self, chunk_index):
@@ -81,60 +96,77 @@ class HDF5ImageCache:
                 raise IndexError(f"{self} only has {self.num_chunks} chunks!")
             return chunk_size
 
+    def get_chunk_slice(self, chunk_index):
+        """Return the slice corresponding to the chunk index"""
+        ch_slice = slice(self.chunk_size * chunk_index,
+                         self.chunk_size * (chunk_index + 1)
+                         )
+        return ch_slice
+
     def iter_chunks(self):
-        size = self.h5ds.shape[0]
         index = 0
         chunk = 0
         while True:
             yield chunk
             chunk += 1
             index += self.chunk_size
-            if index >= size:
+            if index >= self._len:
                 break
 
 
-class ImageCorrCache:
+class HDF5ImageCache(BaseImageChunkCache):
     def __init__(self,
-                 image: HDF5ImageCache,
-                 image_bg: HDF5ImageCache):
-        self.image = image
-        self.image_bg = image_bg
-        self.chunk_size = image.chunk_size
-        self.num_chunks = image.num_chunks
-        self.h5ds = image.h5ds
-        self.shape = image.shape
-        self.chunk_shape = image.chunk_shape
-        #: This is a FILO cache for the corrected image chunks
-        self.cache = collections.OrderedDict()
-        self.cache_size = image.cache_size
+                 h5ds: h5py.Dataset,
+                 chunk_size: int = 1000,
+                 cache_size: int = 2,
+                 boolean: bool = False):
+        """An HDF5 image cache
 
-    def _get_chunk_index_for_index(self, index):
-        if index < 0:
-            index = len(self.h5ds) + index
-        chunk_index = index // self.chunk_size
-        sub_index = index % self.chunk_size
-        return chunk_index, sub_index
+        Deformability cytometry data files commonly contain image stacks
+        that are chunked in various ways. Loading just a single image
+        can be time-consuming, because an entire HDF5 chunk has to be
+        loaded, decompressed and from that one image extracted. The
+        `HDF5ImageCache` class caches the chunks from the HDF5 files
+        into memory, making single-image-access very fast.
+        """
+        super(HDF5ImageCache, self).__init__(
+            shape=h5ds.shape,
+            chunk_size=chunk_size,
+            cache_size=cache_size)
+        # TODO:
+        # - adjust chunking to multiples of the chunks in the dataset
+        #   (which might slightly speed up things)
+        self.h5ds = h5ds
+        self.boolean = boolean
 
-    def __getitem__(self, index):
-        chunk_index, sub_index = self._get_chunk_index_for_index(index)
-        return self.get_chunk(chunk_index)[sub_index]
+        if self._len == 0:
+            warnings.warn(f"Input image '{h5ds.name}' in "
+                          f"file {h5ds.file.filename} has zero length",
+                          EmptyDatasetWarning)
 
-    def __len__(self):
-        return len(self.image)
+    def _get_chunk_data(self, chunk_slice):
+        data = self.h5ds[chunk_slice]
+        if self.boolean:
+            data = np.array(data, dtype=bool)
+        return data
 
-    def get_chunk(self, chunk_index):
-        if chunk_index not in self.cache:
-            data = np.array(
-                self.image.get_chunk(chunk_index), dtype=np.int16) \
-                - self.image_bg.get_chunk(chunk_index)
-            self.cache[chunk_index] = data
-            if len(self.cache) > self.cache_size:
-                # Remove the first item
-                self.cache.popitem(last=False)
-        return self.cache[chunk_index]
 
-    def iter_chunks(self):
-        return self.image.iter_chunks()
+class ImageCorrCache(BaseImageChunkCache):
+    def __init__(self,
+                 image: HDF5ImageCache,
+                 image_bg: HDF5ImageCache):
+        super(ImageCorrCache, self).__init__(
+            shape=image.shape,
+            chunk_size=image.chunk_size,
+            cache_size=image.cache_size)
+        self.image = image
+        self.image_bg = image_bg
+
+    def _get_chunk_data(self, chunk_slice):
+        data = np.array(
+            self.image._get_chunk_data(chunk_slice), dtype=np.int16) \
+           - self.image_bg._get_chunk_data(chunk_slice)
+        return data
 
 
 @functools.cache

dcnum/read/const.py

@@ -1,15 +1,17 @@
-#: Scalar features that apply to all events in a frame
+#: Scalar features that apply to all events in a frame and which are
+#: not computed for individual events.
 PROTECTED_FEATURES = [
-    "bg_med",
+    "bg_off",
     "flow_rate",
     "frame",
     "g_force",
-    "index_online",
     "pressure",
     "temp",
     "temp_amb",
-    "time"
+    "time",
 ]
 
+# User-defined features may be anything, but if the user needs something
+# very specific for the pipeline, having them protected is a nice feature.
 for ii in range(10):
     PROTECTED_FEATURES.append(f"userdef{ii}")