dcnum 0.17.0__py3-none-any.whl → 0.23.1__py3-none-any.whl

dcnum/logic/ctrl.py

@@ -1,5 +1,6 @@
 import collections
 import datetime
+import hashlib
 import json
 import logging
 from logging.handlers import QueueListener
@@ -14,6 +15,7 @@ import traceback
 import uuid
 
 import h5py
+import numpy as np
 
 from ..feat.feat_background.base import get_available_background_methods
 from ..feat.queue_event_extractor import QueueEventExtractor
@@ -21,10 +23,10 @@ from ..feat import gate
 from ..feat import EventExtractorManagerThread
 from ..segm import SegmenterManagerThread, get_available_segmenters
 from ..meta import ppid
-from ..read import HDF5Data
-from .._version import version_tuple
+from ..read import HDF5Data, get_mapping_indices
+from .._version import version, version_tuple
 from ..write import (
-    DequeWriterThread, HDF5Writer, QueueCollectorThread,
+    DequeWriterThread, HDF5Writer, QueueCollectorThread, copy_features,
     copy_metadata, create_with_basins, set_default_filter_kwargs
 )
 
@@ -43,6 +45,7 @@ valid_states = [
     "setup",
     "background",
     "segmentation",
+    "plumbing",
     "cleanup",
     "done",
     "error",
@@ -79,16 +82,16 @@ class DCNumJobRunner(threading.Thread):
         # current job state
         self._state = "init"
         # overall progress [0, 1]
-        self._progress_bg = None
-        self._progress_ex = None
+        self._progress_bg = None  # background
+        self._progress_ex = None  # segmentation
+        self._progress_bn = None  # creating basins
         # segmentation frame rate
         self._segm_rate = 0
 
         # Set up logging
         # General logger for this job
         self.main_logger = logging.getLogger("dcnum")
-        self.main_logger.setLevel(
-            logging.DEBUG if job["debug"] else logging.INFO)
+        self.main_logger.setLevel(job["log_level"])
         # Log file output in target directory
         self.path_log = job["path_out"].with_suffix(".log")
         self.path_log.parent.mkdir(exist_ok=True, parents=True)
@@ -237,8 +240,12 @@ class DCNumJobRunner(threading.Thread):
         # how much fractional time each processing step takes.
         bgw = 4  # fraction of background
         exw = 27  # fraction of segmentation and feature extraction
+        if self.job["basin_strategy"] == "drain":
+            drw = 15  # because data need to be copied
+        else:
+            drw = 1  # just creating the basins in output file
         clw = 1  # fraction of cleanup operations
-        tot = bgw + exw + clw
+        tot = bgw + exw + drw + clw
         progress = 0
         st = self.state
 
@@ -247,15 +254,22 @@ class DCNumJobRunner(threading.Thread):
             # background already computed
             progress += bgw / tot
         elif self._progress_bg is not None:
-            # This is the image count of the input dataset
-            progress += bgw / tot * (self._progress_bg.value / len(self.draw))
+            # This is the image count of the input dataset.
+            progress += self._progress_bg.value * bgw / tot
 
         # segmentation
         if valid_states.index(st) > valid_states.index("segmentation"):
             # segmentation already done
             progress += exw / tot
         elif self._progress_ex is not None:
-            progress += exw / tot * self._progress_ex
+            progress += self._progress_ex * exw / tot
+
+        # draining basins
+        if valid_states.index(st) > valid_states.index("plumbing"):
+            # plumbing already done
+            progress += drw / tot
+        if self._progress_bn is not None:
+            progress += self._progress_bn * drw / tot
 
         if self.state == "done":
             progress = 1
@@ -310,12 +324,23 @@ class DCNumJobRunner(threading.Thread):
             # Whether pipeline hash is invalid.
             ppid.compute_pipeline_hash(**datdict) != dathash
             # Whether the input file is the original output of the pipeline.
-            or len(self.draw) != evyield)
+            or len(self.draw) != evyield
+            # If index mapping is defined, then we always redo the pipeline.
+            # If the pipeline hashes are identical and index mapping is not
+            # None, then both pipelines were done with index mapping.
+            # But applying the same pipeline with index mapping in series
+            # will lead to a different result in the second run (e.g. 1st
+            # pipeline run: take every 2nd event; 2nd pipeline run: take
+            # every second event -> results in every 4th event in output of
+            # second pipeline run).
+            or self.draw.index_mapping is not None
+        )
         # Do we have to recompute the background data? In addition to the
         # hash sanity check above, check the generation, input data,
         # and background pipeline identifiers.
         redo_bg = (
-            (datdict["gen_id"] != self.ppdict["gen_id"])
+            "image_bg" not in self.draw
+            or (datdict["gen_id"] != self.ppdict["gen_id"])
             or (datdict["dat_id"] != self.ppdict["dat_id"])
             or (datdict["bg_id"] != self.ppdict["bg_id"]))
 
@@ -361,16 +386,20 @@ class DCNumJobRunner(threading.Thread):
             # Note any new actions that work on `self.path_temp_in` are not
             # reflected in `self.path_temp_out`.
             self.path_temp_in.rename(self.path_temp_out)
-
-        self.state = "cleanup"
-
-        # The user would normally expect the output file to be something
-        # that is self-contained (copying the file wildly across file
-        # systems and network shares should not impair feature availability).
-        # Therefore, we copy any remaining basin-based features to the
-        # temporary output file.
-        if self.job["no_basins_in_output"]:
-            self.task_transfer_basin_data()
+            # Since no segmentation was done, the output file now does not
+            # contain any events. This is not really what we wanted, but we
+            # can still store all features in the output file if required.
+            if self.job["basin_strategy"] == "drain":
+                orig_feats = []
+                for feat in self.draw.h5["events"].keys():
+                    if isinstance(self.draw.h5["events"][feat], h5py.Dataset):
+                        # copy_features does not support Groups
+                        orig_feats.append(feat)
+                with h5py.File(self.path_temp_out, "a") as h5_dst:
+                    copy_features(h5_src=self.draw.h5,
+                                  h5_dst=h5_dst,
+                                  features=orig_feats,
+                                  mapping=None)
 
         with HDF5Writer(self.path_temp_out) as hw:
             # pipeline metadata
@@ -382,6 +411,10 @@ class DCNumJobRunner(threading.Thread):
             hw.h5.attrs["pipeline:dcnum gate"] = self.ppdict["gate_id"]
             hw.h5.attrs["pipeline:dcnum hash"] = self.pphash
             hw.h5.attrs["pipeline:dcnum yield"] = self.event_count
+            # index mapping information
+            im = self.job.kwargs["data_kwargs"].get("index_mapping", None)
+            dim = HDF5Data.get_ppid_index_mapping(im)
+            hw.h5.attrs["pipeline:dcnum mapping"] = dim
             # regular metadata
             hw.h5.attrs["experiment:event count"] = self.event_count
             hw.h5.attrs["imaging:pixel size"] = self.draw.pixel_size
@@ -419,7 +452,8 @@ class DCNumJobRunner(threading.Thread):
             with h5py.File(self.job["path_in"]) as h5_src:
                 copy_metadata(h5_src=h5_src,
                               h5_dst=hw.h5,
-                              # don't copy basins
+                              # Don't copy basins, we would have to index-map
+                              # them first.
                               copy_basins=False)
             if redo_seg:
                 # Store the correct measurement identifier. This is used to
@@ -429,13 +463,27 @@ class DCNumJobRunner(threading.Thread):
                 # This is the identifier appendix that we use to identify this
                 # dataset. Note that we only override the run identifier when
                 # segmentation did actually take place.
-                mid_ap = "dcn-" + self.pphash[:7]
-                # This is the current measurement identifier (may be empty).
-                mid_cur = hw.h5.attrs.get("experiment:run identifier", "")
+                mid_ap = f"dcn-{self.pphash[:7]}"
+                # This is the current measurement identifier
+                mid_cur = hw.h5.attrs.get("experiment:run identifier")
+                if not mid_cur:
+                    # Compute a measurement identifier from the metadata
+                    m_time = hw.h5.attrs.get("experiment:time", "none")
+                    m_date = hw.h5.attrs.get("experiment:date", "none")
+                    m_sid = hw.h5.attrs.get("setup:identifier", "none")
+                    hasher = hashlib.md5(
+                        f"{m_time}_{m_date}_{m_sid}".encode("utf-8"))
+                    mid_cur = str(uuid.UUID(hex=hasher.hexdigest()))
                 # The new measurement identifier is a combination of both.
                 mid_new = f"{mid_cur}_{mid_ap}" if mid_cur else mid_ap
                 hw.h5.attrs["experiment:run identifier"] = mid_new
 
+        # Handle basin data according to the user's request
+        self.state = "plumbing"
+        self.task_enforce_basin_strategy()
+
+        self.state = "cleanup"
+
         trun = datetime.timedelta(seconds=round(time.monotonic() - time_start))
         self.logger.info(f"Run duration: {str(trun)}")
         self.logger.info(time.strftime("Run stop: %Y-%m-%d-%H.%M.%S",
@@ -477,6 +525,115 @@ class DCNumJobRunner(threading.Thread):
             bic.process()
         self.logger.info("Finished background computation")
 
+    def task_enforce_basin_strategy(self):
+        """Transfer basin data from input files to output if requested
+
+        The user specified the "basin_strategy" keyword argument in
+        `self.job`. If this is set to "drain", then copy all basin
+        information from the input file to the output file. If it
+        is set to "tap", then only create basins in the output file.
+        """
+        self._progress_bn = 0
+        t0 = time.perf_counter()
+        # We need to make sure that the features are correctly attributed
+        # from the input files. E.g. if the input file already has
+        # background images, but we recompute the background images, then
+        # we have to use the data from the recomputed background file.
+        # We achieve this by keeping a specific order and only copying those
+        # features that we don't already have in the output file.
+        feats_raw = [
+            # 1. background data from the temporary input image
+            # (this must come before draw [sic!])
+            [self.dtin.h5, ["image_bg", "bg_off"], "critical"],
+            # 2. frame-based scalar features from the raw input file
+            # (e.g. "temp" or "frame")
+            [self.draw.h5, self.draw.features_scalar_frame, "optional"],
+            # 3. image features from the input file
+            [self.draw.h5, ["image", "image_bg", "bg_off"], "optional"],
+        ]
+        with h5py.File(self.path_temp_out, "a") as hout:
+            hw = HDF5Writer(hout)
+            # First, we have to determine the basin mapping from input to
+            # output. This information is stored by the QueueCollectorThread
+            # in the "basinmap0" feature, ready to be used by us.
+            if "index_unmapped" in hout["events"]:
+                # The unmapped indices enumerate the events in the output file
+                # with indices from the mapped input file. E.g. if for the
+                # first image in the input file, two events are found and for
+                # the second image in the input file, three events are found,
+                # then this would contain [0, 0, 1, 1, 1, ...]. If the index
+                # mapping of the input file was set to slice(1, 100), then the
+                # first image would not be there, and we would have
+                # [1, 1, 1, ...].
+                idx_um = hout["events/index_unmapped"]
+
+                # If we want to convert this to an actual basinmap feature,
+                # then we have to convert those indices to indices that map
+                # to the original input HDF5 file.
+                raw_im = self.draw.index_mapping
+                if raw_im is None:
+                    self.logger.info("Input file mapped with basinmap0")
+                    # Create a hard link to save time and space
+                    hout["events/basinmap0"] = hout["events/index_unmapped"]
+                    basinmap = idx_um
+                else:
+                    basinmap = get_mapping_indices(raw_im)[idx_um]
+                    # Store the mapped basin data in the output file.
+                    hw.store_feature_chunk("basinmap0", basinmap)
+                # We don't need them anymore.
+                del hout["events/index_unmapped"]
+
+                # Note that `size_raw != (len(self.draw))` [sic!]. The former
+                # is the size of the raw dataset and the latter is its mapped
+                # size!
+                size_raw = self.draw.h5.attrs["experiment:event count"]
+                if (len(basinmap) == size_raw
+                        and np.all(basinmap == np.arange(size_raw))):
+                    # This means that the images in the input overlap perfectly
+                    # with the images in the output, i.e. a "copy" segmenter
+                    # was used or something is very reproducible.
+                    # We set basinmap to None to be more efficient.
+                    basinmap = None
+
+            else:
+                # The input is identical to the output, because we are using
+                # the same pipeline identifier.
+                basinmap = None
+
+            for hin, feats, importance in feats_raw:
+                # Only consider features that are available in the input
+                # and that are not already in the output.
+                feats = [f for f in feats
+                         if (f in hin["events"] and f not in hout["events"])]
+                if not feats:
+                    continue
+                elif (self.job["basin_strategy"] == "drain"
+                      or importance == "critical"):
+                    # DRAIN: Copy all features over to the output file.
+                    self.logger.debug(f"Transferring {feats} to output file")
+                    copy_features(h5_src=hin,
+                                  h5_dst=hout,
+                                  features=feats,
+                                  mapping=basinmap)
+                else:
+                    # TAP: Create basins for the "optional" features in the
+                    # output file. Note that the "critical" features never
+                    # reach this case.
+                    self.logger.debug(f"Creating basin for {feats}")
+                    # Relative and absolute paths.
+                    pin = pathlib.Path(hin.filename).resolve()
+                    pout = pathlib.Path(hout.filename).resolve().parent
+                    paths = [pin, os.path.relpath(pin, pout)]
+                    hw.store_basin(name="dcnum basin",
+                                   features=feats,
+                                   mapping=basinmap,
+                                   paths=paths,
+                                   description=f"Created with dcnum {version}",
+                                   )
+                self._progress_bn += 1 / len(feats_raw)
+        t_tot = time.perf_counter() - t0
+        self.logger.info(f"Enforcing basin strategy time: {t_tot:.1f}s")
+
     def task_segment_extract(self):
         self.logger.info("Starting segmentation and feature extraction")
         # Start writer thread
@@ -501,9 +658,9 @@ class DCNumJobRunner(threading.Thread):
             num_slots = 1
             num_extractors = 1
             num_segmenters = 1
-        elif seg_cls.hardware_processor == "cpu":  # CPU segmenter
+        elif seg_cls.hardware_processor == "cpu":  # MPO segmenter
             # We could in principle set the number of slots to one and
-            # jave both number of extractors and number of segmenters set
+            # have both number of extractors and number of segmenters set
             # to the total number of CPUs. However, we would need more RAM
             # (for caching the image data) and we also have more overhead.
             # Having two slots shared between all workers is more efficient.
@@ -511,24 +668,32 @@ class DCNumJobRunner(threading.Thread):
             # Split segmentation and feature extraction workers evenly.
             num_extractors = self.job["num_procs"] // 2
             num_segmenters = self.job["num_procs"] - num_extractors
+            # leave one CPU for the writer and the remaining Threads
+            num_segmenters -= 1
         else:  # GPU segmenter
             num_slots = 3
             num_extractors = self.job["num_procs"]
+            # leave one CPU for the writer and the remaining Threads
+            num_extractors -= 1
             num_segmenters = 1
         num_extractors = max(1, num_extractors)
         num_segmenters = max(1, num_segmenters)
         self.job.kwargs["segmenter_kwargs"]["num_workers"] = num_segmenters
+        self.job.kwargs["segmenter_kwargs"]["debug"] = self.job["debug"]
+        slot_chunks = mp_spawn.Array("i", num_slots, lock=False)
+        slot_states = mp_spawn.Array("u", num_slots, lock=False)
 
-        slot_chunks = mp_spawn.Array("i", num_slots)
-        slot_states = mp_spawn.Array("u", num_slots)
+        self.logger.debug(f"Number of slots: {num_slots}")
+        self.logger.debug(f"Number of segmenters: {num_segmenters}")
+        self.logger.debug(f"Number of extractors: {num_extractors}")
 
-        # Initialize thread
+        # Initialize segmenter manager thread
         thr_segm = SegmenterManagerThread(
             segmenter=seg_cls(**self.job["segmenter_kwargs"]),
             image_data=imdat,
+            bg_off=self.dtin["bg_off"] if "bg_off" in self.dtin else None,
             slot_states=slot_states,
             slot_chunks=slot_chunks,
-            debug=self.job["debug"],
         )
         thr_segm.start()
 
@@ -538,7 +703,7 @@ class DCNumJobRunner(threading.Thread):
             gate=gate.Gate(self.dtin, **self.job["gate_kwargs"]),
             num_extractors=num_extractors,
             log_queue=self.log_queue,
-            log_level=logging.DEBUG if self.job["debug"] else logging.INFO,
+            log_level=self.logger.level,
             )
         fe_kwargs["extract_kwargs"] = self.job["feature_kwargs"]
 
@@ -614,21 +779,6 @@ class DCNumJobRunner(threading.Thread):
 
         self.logger.info("Finished segmentation and feature extraction")
 
-    def task_transfer_basin_data(self):
-        with h5py.File(self.path_temp_out, "a") as hout:
-            hd = HDF5Data(hout)
-            for ii, _ in enumerate(hd.basins):
-                hindat, features = hd.get_basin_data(ii)
-                for feat in features:
-                    if feat not in hout["events"]:
-                        self.logger.debug(
-                            f"Transferring {feat} to output file")
-                        h5py.h5o.copy(src_loc=hindat.h5["events"].id,
-                                      src_name=feat.encode(),
-                                      dst_loc=hout["events"].id,
-                                      dst_name=feat.encode(),
-                                      )
-
 
 def join_thread_helper(thr, timeout, retries, logger, name):
     for _ in range(retries):

dcnum/logic/job.py

@@ -1,9 +1,11 @@
 import collections
 import copy
 import inspect
+import logging
 import multiprocessing as mp
 import pathlib
-from typing import Dict
+from typing import Dict, Literal
+import warnings
 
 from ..feat import QueueEventExtractor
 from ..feat.feat_background.base import get_available_background_methods
@@ -27,10 +29,66 @@ class DCNumPipelineJob:
                  feature_kwargs: Dict = None,
                  gate_code: str = "norm",
                  gate_kwargs: Dict = None,
-                 no_basins_in_output: bool = True,
+                 basin_strategy: Literal["drain", "tap"] = "drain",
+                 no_basins_in_output: bool = None,
                  num_procs: int = None,
+                 log_level: int = logging.INFO,
                  debug: bool = False,
                  ):
+        """Pipeline job recipe
+
+        Parameters
+        ----------
+        path_in: pathlib.Path | str
+            input data path
+        path_out: pathlib.Path | str
+            output data path
+        data_code: str
+            code of input data reader to use
+        data_kwargs: dict
+            keyword arguments for data reader
+        background_code: str
+            code of background data computer to use
+        background_kwargs: dict
+            keyword arguments for background data computer
+        segmenter_code: str
+            code of segmenter to use
+        segmenter_kwargs: dict
+            keyword arguments for segmenter
+        feature_code: str
+            code of feature extractor
+        feature_kwargs: dict
+            keyword arguments for feature extractor
+        gate_code: str
+            code for gating/event filtering class
+        gate_kwargs: dict
+            keyword arguments for gating/event filtering class
+        basin_strategy: str
+            strategy on how to handle event data; In principle, not all
+            events have to be stored in the output file if basins are
+            defined, linking back to the original file.
+            - You can "drain" all basins which means that the output file
+              will contain all features, but will also be very big.
+            - You can "tap" the basins, including the input file, which means
+              that the output file will be comparatively small.
+        no_basins_in_output: bool
+            Deprecated
+        num_procs: int
+            Number of processes to use
+        log_level: int
+            Logging level to use.
+        debug: bool
+            Whether to set logging level to "DEBUG" and
+            use threads instead of processes
+        """
+        if no_basins_in_output is not None:
+            warnings.warn("The `no_basins_in_output` keyword argument is "
+                          "deprecated. Please use `basin_strategy` instead.")
+            if no_basins_in_output:
+                basin_strategy = "drain"
+            else:
+                basin_strategy = "tap"
+
         #: initialize keyword arguments for this job
         self.kwargs = {}
         spec = inspect.getfullargspec(DCNumPipelineJob.__init__)
@@ -51,6 +109,9 @@ class DCNumPipelineJob:
         if path_out is None:
             pin = pathlib.Path(path_in)
             path_out = pin.with_name(pin.stem + "_dcn.rtdc")
+        # Set logging level to DEBUG in debugging mode
+        if self.kwargs["debug"]:
+            self.kwargs["log_level"] = logging.DEBUG
         self.kwargs["path_out"] = pathlib.Path(path_out)
         # Set default mask kwargs for segmenter
         self.kwargs["segmenter_kwargs"].setdefault("kwargs_mask", {})

dcnum/logic/json_encoder.py

@@ -13,5 +13,7 @@ class ExtendedJSONEncoder(json.JSONEncoder):
             return int(obj)
         elif isinstance(obj, np.bool_):
             return bool(obj)
+        elif isinstance(obj, slice):
+            return "PYTHON-SLICE", (obj.start, obj.stop, obj.step)
         # Let the base class default method raise the TypeError
         return json.JSONEncoder.default(self, obj)

dcnum/meta/ppid.py

@@ -10,7 +10,7 @@ import warnings
 
 #: Increment this string if there are breaking changes that make
 #: previous pipelines unreproducible.
-DCNUM_PPID_GENERATION = "7"
+DCNUM_PPID_GENERATION = "10"
 
 
 class ClassWithPPIDCapabilities(Protocol):
@@ -59,7 +59,9 @@ def convert_to_dtype(value, dtype):
 
 
 def get_class_method_info(class_obj: ClassWithPPIDCapabilities,
-                          static_kw_methods: List = None):
+                          static_kw_methods: List = None,
+                          static_kw_defaults: Dict = None,
+                          ):
     """Return dictionary of class info with static keyword methods docs
 
     Parameters
@@ -69,7 +71,16 @@ def get_class_method_info(class_obj: ClassWithPPIDCapabilities,
     static_kw_methods: list of callable
         The methods to inspect; all kwargs-only keyword arguments
         are extracted.
+    static_kw_defaults: dict
+        If a key in this dictionary matches an item in `static_kw_methods`,
+        then these are the default values returned in the "defaults"
+        dictionary. This is used in cases where a base class does
+        implement some annotations, but the subclass does not actually
+        use them, because e.g. they are taken from a property such as is
+        the case for the mask postprocessing of segmenter classes.
     """
+    if static_kw_defaults is None:
+        static_kw_defaults = {}
     doc = class_obj.__doc__ or class_obj.__init__.__doc__
     info = {
         "code": class_obj.get_ppid_code(),
@@ -82,7 +93,10 @@ def get_class_method_info(class_obj: ClassWithPPIDCapabilities,
         for mm in static_kw_methods:
             meth = getattr(class_obj, mm)
             spec = inspect.getfullargspec(meth)
-            defau[mm] = spec.kwonlydefaults or {}
+            if mm_defaults := static_kw_defaults.get(mm):
+                defau[mm] = mm_defaults
+            else:
+                defau[mm] = spec.kwonlydefaults or {}
             annot[mm] = spec.annotations
         info["defaults"] = defau
         info["annotations"] = annot

dcnum/read/__init__.py

@@ -2,3 +2,4 @@
 from .cache import md5sum
 from .const import PROTECTED_FEATURES
 from .hdf5_data import HDF5Data, HDF5ImageCache, concatenated_hdf5_data
+from .mapped import get_mapping_indices, get_mapped_object