dcnum 0.25.6__py3-none-any.whl → 0.25.8__py3-none-any.whl

dcnum/logic/json_encoder.py

@@ -7,6 +7,15 @@ import numpy as np
 
 class ExtendedJSONEncoder(json.JSONEncoder):
     def default(self, obj):
+        """Extended JSON encoder for the **dcnum** logic
+
+        This JSON encoder can handle the following additional objects:
+
+        - ``pathlib.Path``
+        - integer numbers
+        - ``numpy`` boolean
+        - slices (via "PYTHON-SLICE" identifier)
+        """
         if isinstance(obj, pathlib.Path):
             return str(obj)
         elif isinstance(obj, numbers.Integral):

dcnum/meta/paths.py

@@ -1,6 +1,7 @@
 import pathlib
 
 search_path_registry = {}
+"""Dictionary keeping track of user-defined search paths"""
 
 
 def register_search_path(topic: str,

dcnum/meta/ppid.py

@@ -9,9 +9,11 @@ import warnings
 
 import numpy as np
 
-#: Increment this string if there are breaking changes that make
-#: previous pipelines unreproducible.
 DCNUM_PPID_GENERATION = "11"
+"""The dcnum pipeline generation.
+Increment this string if there are breaking changes that make
+previous pipelines unreproducible.
+"""
 
 
 class ClassWithPPIDCapabilities(Protocol):

dcnum/os_env_st.py

@@ -4,7 +4,6 @@ import os
 
 logger = logging.getLogger(__name__)
 
-#: environment variables that set number of threads
 os_env_threading = [
     "MKL_NUM_THREADS",
     "NUMBA_NUM_THREADS",
@@ -14,13 +13,14 @@ os_env_threading = [
     "OPENBLAS_NUM_THREADS",
     "VECLIB_MAXIMUM_THREADS",
 ]
+"""environment variables that define number of threads libraries will use"""
 
 
 class RequestSingleThreaded:
     """Context manager for starting a process with specific environment
 
     When entering the context, the environment variables defined in
-    `os_env_threading` are all set to "1", telling the relevant libraries
+    ``os_env_threading`` are all set to "1", telling the relevant libraries
     that they should work in single-threaded mode.
     When exiting the context, these environment variables are reset to
     their original values (or unset if applicable).

dcnum/read/__init__.py

@@ -2,5 +2,6 @@
 from .cache import md5sum
 from .const import PROTECTED_FEATURES
 from .detect_flicker import detect_flickering
-from .hdf5_data import HDF5Data, HDF5ImageCache, concatenated_hdf5_data
+from .hdf5_data import HDF5Data, HDF5ImageCache
+from .hdf5_concat import concatenated_hdf5_data
 from .mapped import get_mapping_indices, get_mapped_object

dcnum/read/cache.py

@@ -25,8 +25,10 @@ class BaseImageChunkCache(abc.ABC):
         self._dtype = None
         chunk_size = min(shape[0], chunk_size)
         self._len = self.shape[0]
-        #: This is a FILO cache for the chunks
+
         self.cache = collections.OrderedDict()
+        """This is a FILO cache for the chunks"""
+
         self.image_shape = self.shape[1:]
         self.chunk_shape = (chunk_size,) + self.shape[1:]
         self.chunk_size = chunk_size

dcnum/read/const.py

@@ -1,5 +1,3 @@
-#: Scalar features that apply to all events in a frame and which are
-#: not computed for individual events.
 PROTECTED_FEATURES = [
     "bg_off",
     "flow_rate",
@@ -10,6 +8,11 @@ PROTECTED_FEATURES = [
     "temp_amb",
     "time",
 ]
+"""Frame-defined scalar features.
+Scalar features that apply to all events in a frame and which are
+not computed for individual events
+"""
+
 
 # User-defined features may be anything, but if the user needs something
 # very specific for the pipeline, having them protected is a nice feature.

dcnum/read/detect_flicker.py

@@ -17,7 +17,7 @@ def detect_flickering(image_data: np.ndarray | HDF5Data,
     triggering signal.
 
     If flickering is detected, you should use the "sparsemed" background
-    computation with `offset_correction` set to True.
+    computation with ``offset_correction`` set to True.
 
     Parameters
     ----------

dcnum/read/hdf5_concat.py

@@ -0,0 +1,145 @@
+import io
+import pathlib
+import tempfile
+import warnings
+
+import h5py
+import numpy as np
+
+
+from .hdf5_data import HDF5Data
+
+
+def concatenated_hdf5_data(paths: list[pathlib.Path],
+                           path_out: bool | pathlib.Path | None = True,
+                           compute_frame: bool = True,
+                           features: list[str] | None = None):
+    """Return a virtual dataset concatenating all the input paths
+
+    Parameters
+    ----------
+    paths:
+        Path of the input HDF5 files that will be concatenated along
+        the feature axis. The metadata will be taken from the first
+        file.
+    path_out:
+        If `None`, then the dataset is created in memory. If `True`
+        (default), create a file on disk. If a pathlib.Path is specified,
+        the dataset is written to that file. Note that datasets in memory
+        are likely not pickable (so don't use them for multiprocessing).
+    compute_frame:
+        Whether to compute the "events/frame" feature, taking the frame
+        data from the input files and properly incrementing them along
+        the file index.
+    features:
+        List of features to take from the input files.
+
+    Notes
+    -----
+    - If one of the input files does not contain a feature from the first
+      input `paths`, then a `ValueError` is raised. Use the `features`
+      argument to specify which features you need instead.
+    - Basins are not considered.
+    """
+    h5kwargs = {"mode": "w", "libver": "latest"}
+    if isinstance(path_out, (pathlib.Path, str)):
+        h5kwargs["name"] = path_out
+    elif path_out is True:
+        tf = tempfile.NamedTemporaryFile(prefix="dcnum_vc_",
+                                         suffix=".hdf5",
+                                         delete=False)
+        tf.write(b"dummy")
+        h5kwargs["name"] = tf.name
+        tf.close()
+    elif path_out is None:
+        h5kwargs["name"] = io.BytesIO()
+    else:
+        raise ValueError(
+            f"Invalid type for `path_out`: {type(path_out)} ({path_out}")
+
+    if len(paths) == 0:
+        raise ValueError("Please specify at least one file in `paths`!")
+    elif len(paths) == 1:
+        warnings.warn("Only one file passed to `concatenated_hdf5_data`; this "
+                      "is equivalent to using `HDF5Data`, but slower.")
+
+    frames = []
+
+    with h5py.File(**h5kwargs) as hv:
+        # determine the sizes of the input files
+        shapes = {}
+        dtypes = {}
+        size = 0
+        for ii, pp in enumerate(paths):
+            pp = pathlib.Path(pp).resolve()
+            with h5py.File(pp, libver="latest") as h5:
+                # get all feature keys
+                featsi = sorted(h5["events"].keys())
+                # get metadata
+                if ii == 0:
+                    meta = dict(h5.attrs)
+                    if not features:
+                        features = featsi
+                # make sure number of features are consistent
+                if not set(features) <= set(featsi):
+                    raise ValueError(
+                        f"File {pp} contains more features than {paths[0]}!")
+                # populate shapes for all features
+                for feat in features:
+                    if not isinstance(h5["events"][feat], h5py.Dataset):
+                        warnings.warn(
+                            f"Ignoring {feat}; not implemented yet!")
+                        continue
+                    if feat in ["frame", "time"]:
+                        continue
+                    shapes.setdefault(feat, []).append(
+                        h5["events"][feat].shape)
+                    if ii == 0:
+                        dtypes[feat] = h5["events"][feat].dtype
+                # increment size
+                size += h5["events"][features[0]].shape[0]
+                # remember the frame feature if requested
+                if compute_frame:
+                    frames.append(h5["events/frame"][:])
+
+        # write metadata
+        hv.attrs.update(meta)
+
+        # Create the virtual datasets
+        for feat in shapes:
+            if len(shapes[feat][0]) == 1:
+                # scalar feature
+                shape = (sum([sh[0] for sh in shapes[feat]]))
+            else:
+                # non-scalar feature
+                length = (sum([sh[0] for sh in shapes[feat]]))
+                shape = list(shapes[feat][0])
+                shape[0] = length
+                shape = tuple(shape)
+            layout = h5py.VirtualLayout(shape=shape, dtype=dtypes[feat])
+            loc = 0
+            for jj, pp in enumerate(paths):
+                vsource = h5py.VirtualSource(pp, f"events/{feat}",
+                                             shape=shapes[feat][jj])
+                cursize = shapes[feat][jj][0]
+                layout[loc:loc+cursize] = vsource
+                loc += cursize
+            hv.create_virtual_dataset(f"/events/{feat}", layout, fillvalue=0)
+
+        if compute_frame:
+            # concatenate frames and store in dataset
+            frame_concat = np.zeros(size, dtype=np.uint64)
+            locf = 0  # indexing location
+            prevmax = 0  # maximum frame number stored so far in array
+            for fr in frames:
+                offset = prevmax + 1 - fr[0]
+                frame_concat[locf:locf+fr.size] = fr + offset
+                locf += fr.size
+                prevmax = fr[-1] + offset
+            hv.create_dataset("/events/frame", data=frame_concat)
+
+        # write metadata
+        hv.attrs["experiment:event count"] = size
+
+    data = HDF5Data(h5kwargs["name"])
+    return data

dcnum/read/hdf5_data.py

@@ -1,11 +1,9 @@
 from __future__ import annotations
 
 import hashlib
-import io
 import json
 import numbers
 import pathlib
-import tempfile
 from typing import Dict, BinaryIO, List
 import uuid
 import warnings
@@ -160,7 +158,10 @@ class HDF5Data:
         if not hasattr(self, "h5"):
             self.h5 = None
 
-        self.path = state["path"]
+        path = state["path"]
+        if isinstance(path, str):
+            path = pathlib.Path(path)
+        self.path = path
 
         self.md5_5m = state["md5_5m"]
         if self.md5_5m is None:
@@ -552,136 +553,10 @@ class HDF5Data:
         return self._keys
 
 
-def concatenated_hdf5_data(paths: List[pathlib.Path],
-                           path_out: True | pathlib.Path | None = True,
-                           compute_frame: bool = True,
-                           features: List[str] | None = None):
-    """Return a virtual dataset concatenating all the input paths
-
-    Parameters
-    ----------
-    paths:
-        Path of the input HDF5 files that will be concatenated along
-        the feature axis. The metadata will be taken from the first
-        file.
-    path_out:
-        If `None`, then the dataset is created in memory. If `True`
-        (default), create a file on disk. If a pathlib.Path is specified,
-        the dataset is written to that file. Note that datasets in memory
-        are likely not pickable (so don't use them for multiprocessing).
-    compute_frame:
-        Whether to compute the "events/frame" feature, taking the frame
-        data from the input files and properly incrementing them along
-        the file index.
-    features:
-        List of features to take from the input files.
-
-    Notes
-    -----
-    - If one of the input files does not contain a feature from the first
-      input `paths`, then a `ValueError` is raised. Use the `features`
-      argument to specify which features you need instead.
-    - Basins are not considered.
-    """
-    h5kwargs = {"mode": "w", "libver": "latest"}
-    if isinstance(path_out, (pathlib.Path, str)):
-        h5kwargs["name"] = path_out
-    elif path_out is True:
-        tf = tempfile.NamedTemporaryFile(prefix="dcnum_vc_",
-                                         suffix=".hdf5",
-                                         delete=False)
-        tf.write(b"dummy")
-        h5kwargs["name"] = tf.name
-        tf.close()
-    elif path_out is None:
-        h5kwargs["name"] = io.BytesIO()
-    else:
-        raise ValueError(
-            f"Invalid type for `path_out`: {type(path_out)} ({path_out}")
-
-    if len(paths) == 0:
-        raise ValueError("Please specify at least one file in `paths`!")
-    elif len(paths) == 1:
-        warnings.warn("Only one file passed to `concatenated_hdf5_data`; this "
-                      "is equivalent to using `HDF5Data`, but slower.")
-
-    frames = []
-
-    with h5py.File(**h5kwargs) as hv:
-        # determine the sizes of the input files
-        shapes = {}
-        dtypes = {}
-        size = 0
-        for ii, pp in enumerate(paths):
-            pp = pathlib.Path(pp).resolve()
-            with h5py.File(pp, libver="latest") as h5:
-                # get all feature keys
-                featsi = sorted(h5["events"].keys())
-                # get metadata
-                if ii == 0:
-                    meta = dict(h5.attrs)
-                    if not features:
-                        features = featsi
-                # make sure number of features are consistent
-                if not set(features) <= set(featsi):
-                    raise ValueError(
-                        f"File {pp} contains more features than {paths[0]}!")
-                # populate shapes for all features
-                for feat in features:
-                    if not isinstance(h5["events"][feat], h5py.Dataset):
-                        warnings.warn(
-                            f"Ignoring {feat}; not implemented yet!")
-                        continue
-                    if feat in ["frame", "time"]:
-                        continue
-                    shapes.setdefault(feat, []).append(
-                        h5["events"][feat].shape)
-                    if ii == 0:
-                        dtypes[feat] = h5["events"][feat].dtype
-                # increment size
-                size += h5["events"][features[0]].shape[0]
-                # remember the frame feature if requested
-                if compute_frame:
-                    frames.append(h5["events/frame"][:])
-
-        # write metadata
-        hv.attrs.update(meta)
-
-        # Create the virtual datasets
-        for feat in shapes:
-            if len(shapes[feat][0]) == 1:
-                # scalar feature
-                shape = (sum([sh[0] for sh in shapes[feat]]))
-            else:
-                # non-scalar feature
-                length = (sum([sh[0] for sh in shapes[feat]]))
-                shape = list(shapes[feat][0])
-                shape[0] = length
-                shape = tuple(shape)
-            layout = h5py.VirtualLayout(shape=shape, dtype=dtypes[feat])
-            loc = 0
-            for jj, pp in enumerate(paths):
-                vsource = h5py.VirtualSource(pp, f"events/{feat}",
-                                             shape=shapes[feat][jj])
-                cursize = shapes[feat][jj][0]
-                layout[loc:loc+cursize] = vsource
-                loc += cursize
-            hv.create_virtual_dataset(f"/events/{feat}", layout, fillvalue=0)
-
-        if compute_frame:
-            # concatenate frames and store in dataset
-            frame_concat = np.zeros(size, dtype=np.uint64)
-            locf = 0  # indexing location
-            prevmax = 0  # maximum frame number stored so far in array
-            for fr in frames:
-                offset = prevmax + 1 - fr[0]
-                frame_concat[locf:locf+fr.size] = fr + offset
-                locf += fr.size
-                prevmax = fr[-1] + offset
-            hv.create_dataset("/events/frame", data=frame_concat)
-
-        # write metadata
-        hv.attrs["experiment:event count"] = size
-
-    data = HDF5Data(h5kwargs["name"])
-    return data
+def concatenated_hdf5_data(*args, **kwargs):
+    warnings.warn(
+        "Please use `dcnum.read.hdf5_concat.concatenated_hdf5_data`. "
+        "Accessing this method via `dcnum.read.hdf5_data` is deprecated.",
+        DeprecationWarning)
+    from . import hdf5_concat
+    return hdf5_concat.concatenated_hdf5_data(*args, **kwargs)

dcnum/segm/segm_torch/segm_torch_base.py

@@ -52,9 +52,10 @@ class TorchSegmenterBase(Segmenter):
         segmenter_kwargs: dict
             Keyword arguments for the segmenter
         meta: dict
-            Dictionary of metadata from an :class:`HDF5Data` instance
+            Dictionary of metadata from an :class:`.hdf5_data.HDF5Data`
+             instance
         logs: dict
-            Dictionary of logs from an :class:`HDF5Data` instance
+            Dictionary of logs from an :class:`.hdf5_data.HDF5Data` instance
 
         Returns
         -------

dcnum/segm/segm_torch/torch_postproc.py

@@ -11,6 +11,7 @@ def postprocess_masks(masks,
     """Postprocess mask images from ML segmenters
 
     The transformation includes:
+
     - Revert the cropping and padding operations done in
       :func:`.preprocess_images` by padding with zeros and cropping.
     - If the original image shape is larger than the mask image shape,

dcnum/segm/segm_torch/torch_preproc.py

@@ -11,6 +11,7 @@ def preprocess_images(images: np.ndarray,
     """Transform image data to something torch models expect
 
     The transformation includes:
+
     - normalization (division by 255, subtraction of mean, division by std)
     - cropping and padding of the input images to `image_shape`. For padding,
       the median of each *individual* image is used.

dcnum/segm/segmenter.py

@@ -26,17 +26,23 @@ class SegmenterNotApplicableError(BaseException):
 
 
 class Segmenter(abc.ABC):
-    #: Required hardware ("cpu" or "gpu") defined in first-level subclass.
     hardware_processor = "none"
-    #: Whether to enable mask post-processing. If disabled, you should
-    #: make sure that your mask is properly defined and cleaned or you
-    #: have to call `process_mask` in your `segment_algorithm` implementation.
+    """Required hardware ("cpu" or "gpu") defined in first-level subclass."""
+
     mask_postprocessing = True
-    #: Default keyword arguments for mask post-processing. See `process_mask`
-    #: for available options.
+    """Whether to enable mask post-processing.
+    If disabled, you should make sure that your mask is properly defined
+    and cleaned or you have to call `process_mask` in your
+    ``segment_algorithm`` implementation.
+    """
+
     mask_default_kwargs = {}
-    #: If the segmenter requires a background-corrected image, set this to True
+    """Default keyword arguments for mask post-processing.
+    See `process_mask` for available options.
+    """
+
     requires_background_correction = False
+    """Whether the segmenter requires a background-corrected image"""
 
     def __init__(self,
                  *,
@@ -46,10 +52,11 @@ class Segmenter(abc.ABC):
         """Base segmenter class
 
         This is the base segmenter class for the multiprocessing operation
-        segmenter :class:`.MPOSegmenter` (multiple subprocesses are spawned
-        and each of them works on a queue of images) and the single-threaded
-        operation segmenter :class:`.STOSegmenter` (e.g. for batch
-        segmentation on a GPU).
+        segmenter :class:`.segmenter_mpo.MPOSegmenter` (multiple
+        subprocesses are spawned and each of them works on a queue of images)
+        and the single-threaded operation segmenter
+        :class:`.segmenter_sto.STOSegmenter` (e.g. for batch segmentation on
+        a GPU).
 
         Parameters
         ----------
@@ -64,12 +71,15 @@ class Segmenter(abc.ABC):
         self.logger = logging.getLogger(__name__).getChild(
             self.__class__.__name__)
         spec = inspect.getfullargspec(self.segment_algorithm)
-        #: custom keyword arguments for the subclassing segmenter
+
         self.kwargs = spec.kwonlydefaults or {}
+        """custom keyword arguments for the subclassing segmenter"""
+
         self.kwargs.update(kwargs)
 
-        #: keyword arguments for mask post-processing
         self.kwargs_mask = {}
+        """keyword arguments for mask post-processing"""
+
         if self.mask_postprocessing:
             spec_mask = inspect.getfullargspec(self.process_mask)
             self.kwargs_mask.update(spec_mask.kwonlydefaults or {})
@@ -108,7 +118,7 @@ class Segmenter(abc.ABC):
             KEY:KW_APPROACH:KW_MASK
 
         Where KEY is e.g. "legacy" or "watershed", and KW_APPROACH is a
-        list of keyword arguments for `segment_algorithm`, e.g.::
+        list of keyword arguments for ``segment_algorithm``, e.g.::
 
             thresh=-6^blur=0
 
@@ -296,10 +306,10 @@ class Segmenter(abc.ABC):
 
     @functools.cache
     def segment_algorithm_wrapper(self):
-        """Wraps `self.segment_algorithm` to only accept an image
+        """Wraps ``self.segment_algorithm`` to only accept an image
 
-        The static method `self.segment_algorithm` may optionally accept
-        keyword arguments `self.kwargs`. This wrapper returns the
+        The static method ``self.segment_algorithm`` may optionally accept
+        keyword arguments ``self.kwargs``. This wrapper returns the
         wrapped method that only accepts the image as an argument. This
         makes sense if you want to unify
         """
@@ -336,7 +346,7 @@ class Segmenter(abc.ABC):
             additional background offset values that should be subtracted
             from the image data before segmentation. Should only be
             used in combination with segmenters that have
-            `requires_background_correction` set to True.
+            ``requires_background_correction`` set to True.
         """
         images = image_data.get_chunk(chunk)
         if bg_off is not None:
@@ -364,9 +374,10 @@ class Segmenter(abc.ABC):
         segmenter_kwargs: dict
             Keyword arguments for the segmenter
         meta: dict
-            Dictionary of metadata from an :class:`HDF5Data` instance
+            Dictionary of metadata from an :class:`.hdf5_data.HDF5Data`
+            instance
         logs: dict
-            Dictionary of logs from an :class:`HDF5Data` instance
+            Dictionary of logs from an :class:`.hdf5_data.HDF5Data` instance
 
         Returns
         -------

dcnum/segm/segmenter_manager_thread.py

@@ -36,8 +36,8 @@ class SegmenterManagerThread(threading.Thread):
             its job for a slot, the slot value will be set to "e" (for
             "task is with feature extractor").
         slot_chunks:
-            For each slot in `slot_states`, this shared array defines
-            on which chunk in `image_data` the segmentation took place.
+            For each slot in ``slot_states``, this shared array defines
+            on which chunk in ``image_data`` the segmentation took place.
         bg_off:
             1d array containing additional background image offset values
             that are added to each background image before subtraction
@@ -45,10 +45,10 @@ class SegmenterManagerThread(threading.Thread):
 
         Notes
         -----
-        This manager keeps a list `labels_list` which enumerates the
-        slots just like `slot_states` and `slot_chunks` do. For each
+        This manager keeps a list ``labels_list`` which enumerates the
+        slots just like ``slot_states` and ``slot_chunks`` do. For each
         slot, this list contains the labeled image data (integer-valued)
-        for the input `image_data` chunks.
+        for the input ``image_data`` chunks.
 
         The working principle of this `SegmenterManagerThread` allows
         the user to define a fixed number of slots on which the segmenter
@@ -61,21 +61,28 @@ class SegmenterManagerThread(threading.Thread):
         super(SegmenterManagerThread, self).__init__(
               name="SegmenterManager", *args, **kwargs)
         self.logger = logging.getLogger("dcnum.segm.SegmenterManagerThread")
-        #: Segmenter instance
+
         self.segmenter = segmenter
-        #: Image data which is being segmented
+        """Segmenter instance"""
+
         self.image_data = image_data
-        #: Additional, optional background offset
+        """Image data which is being segmented"""
+
         self.bg_off = (
             bg_off if self.segmenter.requires_background_correction else None)
-        #: Slot states
+        """Additional, optional background offset"""
+
         self.slot_states = slot_states
-        #: Current slot chunk index for the slot states
+        """Slot states"""
+
         self.slot_chunks = slot_chunks
-        #: List containing the segmented labels of each slot
+        """Current slot chunk index for the slot states"""
+
         self.labels_list = [None] * len(self.slot_states)
-        #: Time counter for segmentation
+        """List containing the segmented labels of each slot"""
+
         self.t_count = 0
+        """Time counter for segmentation"""
 
     def run(self):
         num_slots = len(self.slot_states)

dcnum/segm/segmenter_mpo.py

@@ -35,7 +35,7 @@ class MPOSegmenter(Segmenter, abc.ABC):
         debug: bool
             Debugging parameters
         kwargs:
-            Additional, optional keyword arguments for `segment_algorithm`
+            Additional, optional keyword arguments for ``segment_algorithm``
             defined in the subclass.
         """
         super(MPOSegmenter, self).__init__(kwargs_mask=kwargs_mask,
@@ -145,7 +145,7 @@ class MPOSegmenter(Segmenter, abc.ABC):
         """Perform batch segmentation of `images`
 
         Before segmentation, an optional background offset correction with
-        `bg_off` is performed. After segmentation, mask postprocessing is
+        ``bg_off`` is performed. After segmentation, mask postprocessing is
         performed according to the class definition.
 
         Parameters
@@ -264,7 +264,7 @@ class MPOSegmenter(Segmenter, abc.ABC):
         """Return the integer label image for an input image
 
         Before segmentation, an optional background offset correction with
-        `bg_off` is performed. After segmentation, mask postprocessing is
+        ``bg_off`` is performed. After segmentation, mask postprocessing is
         performed according to the class definition.
         """
         segm_wrap = self.segment_algorithm_wrapper()
@@ -296,7 +296,7 @@ class MPOSegmenterWorker:
 
         Parameters
         ----------
-        segmenter: MPOSegmenter
+        segmenter: .segmenter_mpo.MPOSegmenter
             The segmentation instance
         sl_start: int
             Start of slice of input array to process