dcnum 0.16.1__py3-none-any.whl → 0.16.3__py3-none-any.whl

dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.16.1'
-__version_tuple__ = version_tuple = (0, 16, 1)
+__version__ = version = '0.16.3'
+__version_tuple__ = version_tuple = (0, 16, 3)

dcnum/feat/event_extractor_manager_thread.py

@@ -46,8 +46,6 @@ class EventExtractorManagerThread(threading.Thread):
         """
         super(EventExtractorManagerThread, self).__init__(
               name="EventExtractorManager", *args, **kwargs)
-        if debug:
-            fe_kwargs["close_queues"] = False
         self.logger = logging.getLogger(
             "dcnum.feat.EventExtractorManagerThread")
         #: Keyword arguments for class:`.EventExtractor`
@@ -83,9 +81,9 @@ class EventExtractorManagerThread(threading.Thread):
                    for _ in range(self.num_workers)]
         [w.start() for w in workers]
 
+        num_slots = len(self.slot_states)
         chunks_processed = 0
         while True:
-            num_slots = len(self.slot_states)
             cur_slot = 0
             unavailable_slots = 0
             # Check all slots for segmented labels
@@ -95,8 +93,10 @@ class EventExtractorManagerThread(threading.Thread):
                 # - "s" the extractor processed the data and is waiting
                 #   for the segmenter
                 if self.slot_states[cur_slot] == "e":
+                    # The segmenter has something for us in this slot.
                     break
                 else:
+                    # Try another slot.
                     unavailable_slots += 1
                     cur_slot = (cur_slot + 1) % num_slots
                 if unavailable_slots >= num_slots:
@@ -136,15 +136,31 @@ class EventExtractorManagerThread(threading.Thread):
             if chunks_processed == self.data.image.num_chunks:
                 break
 
-        self.logger.debug("Waiting for event_queue to empty.")
         # Wait until the event queue is empty.
+        self.logger.debug("Waiting for event_queue to empty.")
         event_queue = self.fe_kwargs["event_queue"]
         while not event_queue.empty():
             # The collector thread is still sorting things out. Wait
             # before joining the threads.
-            time.sleep(.1)
+            time.sleep(.05)
+
+        # Wait until log queue is empty
+        self.logger.debug("Waiting for log_queue to empty.")
+        log_queue = self.fe_kwargs["log_queue"]
+        while not log_queue.empty():
+            time.sleep(.05)
+
+        inv_masks = self.fe_kwargs["invalid_mask_counter"].value
+        if inv_masks:
+            self.logger.info(f"Encountered {inv_masks} invalid masks.")
+            inv_frac = inv_masks / len(self.data)
+            if inv_frac > 0.005:  # warn above one half percent
+                self.logger.warning(f"Discarded {inv_frac:.1%} of the masks. "
+                                    f"Please check segmenter applicability.")
+
         self.logger.debug("Requesting extraction workers to join.")
         self.fe_kwargs["finalize_extraction"].value = True
         [w.join() for w in workers]
+
         self.logger.debug("Finished extraction.")
         self.logger.info(f"Extraction time: {self.t_count:.1f}s")

dcnum/feat/feat_background/base.py

@@ -7,12 +7,16 @@ import uuid
 import warnings
 
 import h5py
-import hdf5plugin
 import numpy as np
 
 from ...meta import ppid
 from ...read import HDF5Data
-from ...write import create_with_basins
+from ...write import create_with_basins, set_default_filter_kwargs
+
+
+# All subprocesses should use 'spawn' to avoid issues with threads
+# and 'fork' on POSIX systems.
+mp_spawn = mp.get_context('spawn')
 
 
 class Background(abc.ABC):
@@ -55,12 +59,14 @@ class Background(abc.ABC):
         self.kwargs.update(kwargs)
 
         if num_cpus is None:
-            num_cpus = mp.cpu_count()
+            num_cpus = mp_spawn.cpu_count()
         #: number of CPUs used
         self.num_cpus = num_cpus
 
-        #: number of frames
-        self.event_count = None
+        #: number of images in the input data
+        self.image_count = None
+        #: number of images that have been processed
+        self.image_proc = mp_spawn.Value("L", 0)
 
         #: HDF5Data instance for input data
         self.hdin = None
@@ -93,7 +99,7 @@ class Background(abc.ABC):
         #: shape of event images
         self.image_shape = self.input_data[0].shape
         #: total number of events
-        self.event_count = len(self.input_data)
+        self.image_count = len(self.input_data)
 
         if self.h5out is None:
             if not output_path.exists():
@@ -105,19 +111,15 @@ class Background(abc.ABC):
             self.h5out = h5py.File(output_path, "a", libver="latest")
 
         # Initialize background data
-        if compress:
-            compression_kwargs = hdf5plugin.Zstd(clevel=5)
-        else:
-            compression_kwargs = {}
+        ds_kwargs = set_default_filter_kwargs(compression=compress)
         h5bg = self.h5out.require_dataset(
             "events/image_bg",
             shape=self.input_data.shape,
             dtype=np.uint8,
-            chunks=(min(100, self.event_count),
+            chunks=(min(100, self.image_count),
                     self.image_shape[0],
                     self.image_shape[1]),
-            fletcher32=True,
-            **compression_kwargs,
+            **ds_kwargs,
         )
         h5bg.attrs.create('CLASS', np.string_('IMAGE'))
         h5bg.attrs.create('IMAGE_VERSION', np.string_('1.2'))
@@ -191,6 +193,13 @@ class Background(abc.ABC):
                                      ppid=pp_check_user_kwargs)
         return kwargs
 
+    def get_progress(self):
+        """Return progress of background computation, float in [0,1]"""
+        if self.image_count == 0:
+            return 0.
+        else:
+            return self.image_proc.value / self.image_count
+
     def process(self):
         self.process_approach()
         bg_ppid = self.get_ppid()

dcnum/feat/feat_background/bg_roll_median.py

@@ -1,16 +1,10 @@
-import multiprocessing as mp
 import queue
 import time
 
 import numpy as np
 from scipy import ndimage
 
-from .base import Background
-
-
-# All subprocesses should use 'spawn' to avoid issues with threads
-# and 'fork' on POSIX systems.
-mp_spawn = mp.get_context('spawn')
+from .base import mp_spawn, Background
 
 
 class BackgroundRollMed(Background):
@@ -152,9 +146,9 @@ class BackgroundRollMed(Background):
         stop_in = (batch_index + 1) * self.batch_size + self.kernel_size
         stop_out = (batch_index + 1) * self.batch_size
 
-        if stop_in > self.event_count:
-            stop_in = self.event_count
-            stop_out = self.event_count - self.kernel_size
+        if stop_in > self.image_count:
+            stop_in = self.image_count
+            stop_out = self.image_count - self.kernel_size
 
         slice_in = slice(start, stop_in)
         slice_out = slice(start, stop_out)
@@ -175,16 +169,14 @@ class BackgroundRollMed(Background):
 
     def process_approach(self):
         """Perform median computation on entire input data"""
-        num_steps = int(np.ceil(self.event_count / self.batch_size))
+        num_steps = int(np.ceil(self.image_count / self.batch_size))
         for ii in range(num_steps):
-            print(f"Computing background {ii/num_steps*100:.0f}%",
-                  end="\r", flush=True)
             self.process_next_batch()
         # Set the remaining kernel_size median values to the last one
         last_image = self.h5out["events/image_bg"][-self.kernel_size-1]
         for ii in range(self.kernel_size):
-            self.h5out["events/image_bg"][self.event_count-ii-1] = last_image
-        print("Computing background 100%    ", flush=True)
+            self.h5out["events/image_bg"][self.image_count-ii-1] = last_image
+        self.image_proc.value = self.image_count
 
     def process_next_batch(self):
         """Process one batch of input data"""
@@ -221,6 +213,7 @@ class BackgroundRollMed(Background):
                                                          *self.image_shape)
 
         self.current_batch += 1
+        self.image_proc.value += self.batch_size
 
 
 class MedianWorker(mp_spawn.Process):

dcnum/feat/feat_background/bg_sparse_median.py

@@ -1,19 +1,15 @@
 import logging
-import multiprocessing as mp
 import queue
 import time
 
 import numpy as np
 from scipy import ndimage
 
-from .base import Background
-
-logger = logging.getLogger(__name__)
+from ...read import HDF5Data
 
+from .base import mp_spawn, Background
 
-# All subprocesses should use 'spawn' to avoid issues with threads
-# and 'fork' on POSIX systems.
-mp_spawn = mp.get_context('spawn')
+logger = logging.getLogger(__name__)
 
 
 class BackgroundSparseMed(Background):
@@ -96,27 +92,28 @@ class BackgroundSparseMed(Background):
         # time axis
         self.time = None
         if self.h5in is not None:
-            if "time" in self.h5in["events"]:
+            hd = HDF5Data(self.h5in)
+            if "time" in hd:
                 # use actual time from dataset
-                self.time = self.h5in["/events/time"][:]
+                self.time = hd["time"][:]
                 self.time -= self.time[0]
-            elif "imaging:frame rate" in self.h5in.attrs:
-                fr = self.h5in.attrs["imaging:frame rate"]
-                if "frame" in self.h5in["/events"]:
+            elif "imaging:frame rate" in hd.meta:
+                fr = hd.meta["imaging:frame rate"]
+                if "frame" in hd:
                     # compute time from frame rate and frame numbers
-                    self.time = self.h5in["/events/frame"] / fr
+                    self.time = hd["frame"] / fr
                     self.time -= self.time[0]
                 else:
                     # compute time using frame rate (approximate)
-                    dur = self.event_count / fr * 1.5
+                    dur = self.image_count / fr * 1.5
                     logger.info(f"Approximating duration: {dur/60:.1f}min")
-                    self.time = np.linspace(0, dur, self.event_count,
+                    self.time = np.linspace(0, dur, self.image_count,
                                             endpoint=True)
         if self.time is None:
             # No HDF5 file or no information therein; Make an educated guess.
-            dur = self.event_count / 3600 * 1.5
+            dur = self.image_count / 3600 * 1.5
             logger.info(f"Guessing duration: {dur/60:.1f}min")
-            self.time = np.linspace(0, dur, self.event_count,
+            self.time = np.linspace(0, dur, self.image_count,
                                     endpoint=True)
 
         #: duration of the measurement
@@ -212,10 +209,7 @@ class BackgroundSparseMed(Background):
 
         # Compute initial background images (populates self.bg_images)
         for ii, ti in enumerate(self.step_times):
-            print(f"Computing background {ii / self.step_times.size:.0%}",
-                  end="\r", flush=True)
             self.process_second(ii, ti)
-        print("Computing background 100%    ", flush=True)
 
         if self.frac_cleansing != 1:
             # The following algorithm finds background images that contain
@@ -277,7 +271,7 @@ class BackgroundSparseMed(Background):
                     f"`thresh_cleansing` or `frac_cleansing`. The new "
                     f"threshold is {thresh_fact / thresh}.")
 
-            logger.info(f"Removed {frac_remove:.2%} of the background series")
+            logger.info(f"Cleansed {frac_remove:.2%}")
             step_times = self.step_times[used]
             bg_images = self.bg_images[used]
         else:
@@ -286,7 +280,7 @@ class BackgroundSparseMed(Background):
             bg_images = self.bg_images
 
         # Assign each frame to a certain background index
-        bg_idx = np.zeros(self.event_count, dtype=int)
+        bg_idx = np.zeros(self.image_count, dtype=int)
         idx0 = 0
         idx1 = None
         for ii in range(len(step_times)):
@@ -298,21 +292,25 @@ class BackgroundSparseMed(Background):
             # Fill up remainder of index array with last entry
             bg_idx[idx1:] = ii
 
+        self.image_proc.value = self.image_count
+
         # Write background data
         pos = 0
         step = 1000
-        while pos < self.event_count:
-            stop = min(pos + step, self.event_count)
+        while pos < self.image_count:
+            stop = min(pos + step, self.image_count)
             cur_slice = slice(pos, stop)
             self.h5out["events/image_bg"][cur_slice] = \
                 bg_images[bg_idx[cur_slice]]
             pos += step
 
-    def process_second(self, ii, second):
+    def process_second(self,
+                       ii: int,
+                       second: float | int):
         idx_start = np.argmin(np.abs(second - self.time))
         idx_stop = idx_start + self.kernel_size
-        if idx_stop >= self.event_count:
-            idx_stop = self.event_count
+        if idx_stop >= self.image_count:
+            idx_stop = self.image_count
             idx_start = max(0, idx_stop - self.kernel_size)
         assert idx_stop - idx_start == self.kernel_size
 
@@ -347,6 +345,8 @@ class BackgroundSparseMed(Background):
 
         self.bg_images[ii] = self.shared_output.reshape(self.image_shape)
 
+        self.image_proc.value = idx_stop
+
 
 class MedianWorkerSingle(mp_spawn.Process):
     def __init__(self, job_queue, counter, shared_input, shared_output,

dcnum/feat/queue_event_extractor.py

@@ -35,7 +35,8 @@ class QueueEventExtractor:
                  feat_nevents: mp.Array,
                  label_array: mp.Array,
                  finalize_extraction: mp.Value,
-                 close_queues: bool = True,
+                 invalid_mask_counter: mp.Value,
+                 log_level: int = logging.INFO,
                  extract_kwargs: dict = None,
                  *args, **kwargs):
         """Base class for event extraction from label images
@@ -66,9 +67,10 @@ class QueueEventExtractor:
         finalize_extraction:
             Shared value indicating whether this worker should stop as
             soon as the `raw_queue` is empty.
-        close_queues: bool
-            Whether to close event and logging queues
-            (set to False in debug mode)
+        invalid_mask_counter:
+            Counts masks labeled as invalid by the feature extractor
+        log_level:
+            Logging level to use
         extract_kwargs:
             Keyword arguments for the extraction process. See the
             keyword-only arguments in
@@ -85,7 +87,13 @@ class QueueEventExtractor:
         self.event_queue = event_queue
         #: queue for logging
         self.log_queue = log_queue
-        self.close_queues = close_queues
+        #: invalid mask counter
+        self.invalid_mask_counter = invalid_mask_counter
+        # Logging needs to be set up after `start` is called, otherwise
+        # it looks like we have the same PID as the parent process. We
+        # are setting up logging in `run`.
+        self.logger = None
+        self.log_level = log_level
         #: Shared array of length `len(data)` into which the number of
         #: events per frame is written.
         self.feat_nevents = feat_nevents
@@ -100,15 +108,12 @@ class QueueEventExtractor:
         extract_kwargs.setdefault("haralick", True)
         #: Feature extraction keyword arguments.
         self.extract_kwargs = extract_kwargs
-        # Logging needs to be set up after `start` is called, otherwise
-        # it looks like we have the same PID as the parent process. We
-        # are setting up logging in `run`.
-        self.logger = None
 
     @staticmethod
     def get_init_kwargs(data: HDF5Data,
                         gate: Gate,
                         log_queue: mp.Queue,
+                        log_level: int = logging.INFO,
                         preselect: None = None,
                         ptp_median: None = None):
         """Get initialization arguments for :cass:`.QueueEventExtractor`
@@ -125,7 +130,9 @@ class QueueEventExtractor:
         gate: HDF5Data
             Gating class to use
         log_queue: mp.Queue
-            Queue for sending log messages
+            Queue the worker uses for sending log messages
+        log_level: int
+            Logging level to use in the worker process
         preselect, ptp_median:
             Deprecated
 
@@ -146,6 +153,7 @@ class QueueEventExtractor:
             warnings.warn("The `ptp_median` argument is deprecated!",
                           DeprecationWarning)
 
+        # Note that the order must be identical to  __init__
         args = collections.OrderedDict()
         args["data"] = data
         args["gate"] = gate
@@ -159,7 +167,8 @@ class QueueEventExtractor:
             np.ctypeslib.ctypes.c_int16,
             int(np.prod(data.image.chunk_shape)))
         args["finalize_extraction"] = mp_spawn.Value("b", False)
-        args["close_queues"] = True
+        args["invalid_mask_counter"] = mp_spawn.Value("L", 0)
+        args["log_level"] = log_level
         return args
 
     def get_events_from_masks(self, masks, data_index, *,
@@ -207,8 +216,7 @@ class QueueEventExtractor:
         # over from gated_events to valid_events. According to our experience
         # invalid events happen rarely though.
         if np.any(invalid):
-            self.logger.info(f"Discarded {np.sum(invalid)} events due to "
-                             "invalid segmentation.")
+            self.invalid_mask_counter.value += np.sum(invalid)
             for key in gated_events:
                 valid_events[key] = gated_events[key][valid]
         else:
@@ -294,17 +302,27 @@ class QueueEventExtractor:
         """Main loop of worker process"""
         # Don't wait for these two queues when joining workers
         self.raw_queue.cancel_join_thread()
-        self.log_queue.cancel_join_thread()
         #: logger sends all logs to `self.log_queue`
         self.logger = logging.getLogger(
             f"dcnum.feat.EventExtractor.{os.getpid()}")
+        self.logger.setLevel(self.log_level)
+        # Clear any handlers that might be set for this logger. This is
+        # important for the case when we are an instance of
+        # EventExtractorThread, because then all handlers from the main
+        # thread are inherited (as opposed to no handlers in the case
+        # of EventExtractorProcess).
+        self.logger.handlers.clear()
         queue_handler = QueueHandler(self.log_queue)
+        queue_handler.setLevel(self.log_level)
         self.logger.addHandler(queue_handler)
-        self.logger.addFilter(DeduplicatingLoggingFilter())
-        self.logger.debug(f"Running {self} in PID {os.getpid()}")
+        self.logger.info("Ready")
 
         mp_array = np.ctypeslib.as_array(
             self.label_array).reshape(self.data.image.chunk_shape)
+
+        # only close queues when we have created them ourselves.
+        close_queues = isinstance(self, EventExtractorProcess)
+
         while True:
             try:
                 chunk_index, label_index = self.raw_queue.get(timeout=.03)
@@ -332,15 +350,21 @@ class QueueEventExtractor:
                     self.event_queue.put((index, events))
 
         self.logger.debug(f"Finalizing `run` for PID {os.getpid()}, {self}")
-        if self.close_queues:
+        if close_queues:
             # Explicitly close the event queue and join it
             self.event_queue.close()
             self.event_queue.join_thread()
             self.logger.debug(f"End of `run` for PID {os.getpid()}, {self}")
+
+        # Make sure everything gets written to the queue.
+        queue_handler.flush()
+
+        if close_queues:
             # Also close the logging queue. Note that not all messages might
             # arrive in the logging queue, since we called `cancel_join_thread`
             # earlier.
             self.log_queue.close()
+            self.log_queue.join_thread()
 
     @classmethod
     def get_ppid_from_kwargs(cls, kwargs):
@@ -362,17 +386,3 @@ class EventExtractorThread(QueueEventExtractor, threading.Thread):
     def __init__(self, *args, **kwargs):
         super(EventExtractorThread, self).__init__(
             name="EventExtractorThread", *args, **kwargs)
-
-
-class DeduplicatingLoggingFilter(logging.Filter):
-    def __init__(self, *args, **kwargs):
-        super(DeduplicatingLoggingFilter, self).__init__(*args, **kwargs)
-        self._records = []
-
-    def filter(self, record):
-        """Return True if the record should be logged"""
-        msg = record.getMessage()
-        logged = msg in self._records
-        if not logged:
-            self._records.append(msg)
-        return not logged

dcnum/logic/ctrl.py

@@ -1,15 +1,18 @@
 import collections
+import datetime
+import json
 import logging
 from logging.handlers import QueueListener
 import multiprocessing as mp
 import os
 import pathlib
+import platform
 import socket
 import threading
 import time
+import traceback
 import uuid
 
-import hdf5plugin
 import h5py
 
 from ..feat.feat_background.base import get_available_background_methods
@@ -19,18 +22,32 @@ from ..feat import EventExtractorManagerThread
 from ..segm import SegmenterManagerThread, get_available_segmenters
 from ..meta import ppid
 from ..read import HDF5Data
+from .._version import version_tuple
 from ..write import (
     DequeWriterThread, HDF5Writer, QueueCollectorThread,
-    copy_metadata, create_with_basins,
+    copy_metadata, create_with_basins, set_default_filter_kwargs
 )
 
 from .job import DCNumPipelineJob
-
+from .json_encoder import ExtendedJSONEncoder
 
 # Force using "spawn" method for multiprocessing, because we are using
 # queues and threads and would end up with race conditions otherwise.
 mp_spawn = mp.get_context("spawn")
 
+#: valid states for a job runnter. The states must be in logical ordern,
+#: not in alphabetical order.
+valid_states = [
+    "created",
+    "init",
+    "setup",
+    "background",
+    "segmentation",
+    "cleanup",
+    "done",
+    "error",
+]
+
 
 class DCNumJobRunner(threading.Thread):
     def __init__(self,
@@ -48,6 +65,7 @@ class DCNumJobRunner(threading.Thread):
             (defaults to hostname)
         """
         super(DCNumJobRunner, self).__init__(*args, **kwargs)
+        self.error_tb = None
         self.job = job
         if tmp_suffix is None:
             tmp_suffix = f"{socket.gethostname()}_{str(uuid.uuid4())[:5]}"
@@ -61,16 +79,16 @@ class DCNumJobRunner(threading.Thread):
         # current job state
         self._state = "init"
         # overall progress [0, 1]
-        self._progress = 0
+        self._progress_bg = None
+        self._progress_ex = None
         # segmentation frame rate
         self._segm_rate = 0
 
         # Set up logging
         # General logger for this job
-        self.logger = logging.getLogger(__name__).getChild(
-            f"Runner-{self.pphash[:5]}")
-        self.logger.setLevel(
-            logging.DEBUG if job["debug"] else logging.WARNING)
+        self.main_logger = logging.getLogger("dcnum")
+        self.main_logger.setLevel(
+            logging.DEBUG if job["debug"] else logging.INFO)
         # Log file output in target directory
         self.path_log = job["path_out"].with_suffix(".log")
         self.path_log.parent.mkdir(exist_ok=True, parents=True)
@@ -81,17 +99,30 @@ class DCNumJobRunner(threading.Thread):
             delay=True,
             errors="ignore",
         )
+        # Set the log file handler level to DEBUG, so it logs everything
+        # presented to it.
+        self._log_file_handler.setLevel(logging.DEBUG)
         fmt = logging.Formatter(
-            "%(asctime)s %(levelname)s %(processName)s/%(threadName)s "
-            + "in %(name)s: %(message)s")
+            fmt="%(asctime)s %(levelname)s %(name)s: %(message)s",
+            datefmt='%H:%M:%S'
+        )
         self._log_file_handler.setFormatter(fmt)
-        self.logger.addHandler(self._log_file_handler)
-        handlers = list(self.logger.handlers)
+        self.main_logger.addHandler(self._log_file_handler)
+        handlers = list(self.main_logger.handlers)
+
         # Queue for subprocesses to log to
         self.log_queue = mp_spawn.Queue()
-        self._qlisten = QueueListener(self.log_queue, *handlers)
+        self._qlisten = QueueListener(self.log_queue, *handlers,
+                                      respect_handler_level=True)
         self._qlisten.start()
 
+        if job["debug"]:
+            self.main_logger.info("Note that in debugging mode, duplicate "
+                                  "log entries may appear (logs that are "
+                                  "recorded via queues).")
+
+        self.logger = logging.getLogger(f"dcnum.Runner-{self.pphash[:2]}")
+
         # Sanity checks
         for os_env in [
             "OMP_NUM_THREADS",
@@ -159,6 +190,16 @@ class DCNumJobRunner(threading.Thread):
         po = pathlib.Path(self.job["path_out"])
         return po.with_name(po.stem + f"_output_{self.tmp_suffix}.rtdc~")
 
+    @property
+    def state(self):
+        return self._state
+
+    @state.setter
+    def state(self, state):
+        if state not in valid_states:
+            raise ValueError(f"Invalid state '{state}' specified!")
+        self._state = state
+
     def close(self, delete_temporary_files=True):
         if self._data_raw is not None:
             self._data_raw.close()
@@ -167,8 +208,8 @@ class DCNumJobRunner(threading.Thread):
             self._data_temp_in.close()
             self._data_temp_in = None
         # clean up logging
-        if self._log_file_handler in self.logger.handlers:
-            self.logger.removeHandler(self._log_file_handler)
+        if self._log_file_handler in self.main_logger.handlers:
+            self.main_logger.removeHandler(self._log_file_handler)
             self._log_file_handler.flush()
             self._log_file_handler.close()
         if self._qlisten is not None:
@@ -184,26 +225,67 @@ class DCNumJobRunner(threading.Thread):
             # We don't have to delete self.path_temp_out, since this one
             # is `rename`d to `self.jon["path_out"]`.
 
-    def join(self, *args, **kwargs):
+    def join(self, delete_temporary_files=True, *args, **kwargs):
         super(DCNumJobRunner, self).join(*args, **kwargs)
         # Close only after join
-        self.close()
+        self.close(delete_temporary_files=delete_temporary_files)
 
     def get_status(self):
+        # Compute the total progress. The following weights indicate
+        # how much fractional time each processing step takes.
+        bgw = 4  # fraction of background
+        exw = 27  # fraction of segmentation and feature extraction
+        clw = 1  # fraction of cleanup operations
+        tot = bgw + exw + clw
+        progress = 0
+        st = self.state
+
+        # background
+        if valid_states.index(st) > valid_states.index("background"):
+            # background already computed
+            progress += bgw / tot
+        elif self._progress_bg is not None:
+            # This is the image count of the input dataset
+            progress += bgw / tot * (self._progress_bg.value / len(self.draw))
+
+        # segmentation
+        if valid_states.index(st) > valid_states.index("segmentation"):
+            # segmentation already done
+            progress += exw / tot
+        elif self._progress_ex is not None:
+            progress += exw / tot * self._progress_ex
+
+        if self.state == "done":
+            progress = 1
+
         return {
-            "progress": self._progress,
+            "progress": progress,
             "segm rate": self._segm_rate,
             "state": self._state,
         }
 
     def run(self):
+        try:
+            self.run_pipeline()
+        except BaseException:
+            self.state = "error"
+            self.error_tb = traceback.format_exc()
+            if not self.is_alive():
+                # Thread has not been started. This means we are not running
+                # in a thread but in the main process. Raise the exception.
+                raise
+
+    def run_pipeline(self):
         """Execute the pipeline job"""
+        time_start = time.monotonic()
+        time_string = time.strftime("%Y-%m-%d-%H.%M.%S", time.gmtime())
+        self.logger.info(f"Run start: {time_string}")
         if self.job["path_out"].exists():
             raise FileExistsError(
                 f"Output file {self.job['path_out']} already exists!")
         # Make sure the output directory exists.
         self.job["path_out"].parent.mkdir(parents=True, exist_ok=True)
-        self._state = "setup"
+        self.state = "setup"
         # First get a list of all pipeline IDs. If the input file has
         # already been processed by dcnum, then we do not have to redo
         # everything.
@@ -223,11 +305,10 @@ class DCNumJobRunner(threading.Thread):
         # The number of events extracted in a potential previous pipeline run.
         evyield = self.draw.h5.attrs.get("pipeline:dcnum yield", -1)
         redo_sanity = (
-             # Whether pipeline hash is invalid.
-             ppid.compute_pipeline_hash(**datdict) != dathash
-             # Whether the input file is the original output of the pipeline.
-             or len(self.draw) != evyield
-        )
+            # Whether pipeline hash is invalid.
+            ppid.compute_pipeline_hash(**datdict) != dathash
+            # Whether the input file is the original output of the pipeline.
+            or len(self.draw) != evyield)
         # Do we have to recompute the background data? In addition to the
         # hash sanity check above, check the generation, input data,
         # and background pipeline identifiers.
@@ -246,7 +327,7 @@ class DCNumJobRunner(threading.Thread):
             or (datdict["feat_id"] != self.ppdict["feat_id"])
             or (datdict["gate_id"] != self.ppdict["gate_id"]))
 
-        self._state = "background"
+        self.state = "background"
 
         if redo_bg:
             # The 'image_bg' feature is written to `self.path_temp_in`.
@@ -255,8 +336,7 @@ class DCNumJobRunner(threading.Thread):
             # (note that `self.path_temp_in` is basin-based).
             self.task_background()
 
-        self._progress = 0.1
-        self._state = "segmentation"
+        self.state = "segmentation"
 
         # We have the input data covered, and we have to run the
         # long-lasting segmentation and feature extraction step.
@@ -280,8 +360,7 @@ class DCNumJobRunner(threading.Thread):
             # reflected in `self.path_temp_out`.
             self.path_temp_in.rename(self.path_temp_out)
 
-        self._progress = 0.95
-        self._state = "cleanup"
+        self.state = "cleanup"
 
         # The user would normally expect the output file to be something
         # that is self-contained (copying the file wildly across file
@@ -304,11 +383,36 @@ class DCNumJobRunner(threading.Thread):
             # regular metadata
             hw.h5.attrs["experiment:event count"] = self.event_count
             hw.h5.attrs["imaging:pixel size"] = self.draw.pixel_size
-            if self.path_log.exists():
-                # Add the log file to the resulting .rtdc file
-                hw.store_log(
-                    time.strftime("dcnum-process-%Y-%m-%d-%H.%M.%S"),
-                    self.path_log.read_text().split("\n"))
+            # Add job information to resulting .rtdc file
+            hw.store_log(f"dcnum-job-{time_string}",
+                         json.dumps({
+                             "dcnum version": version_tuple,
+                             "job": self.job.__getstate__(),
+                             "pipeline": {"identifiers": self.ppdict,
+                                          "hash": self.pphash,
+                                          },
+                             "python": {
+                                 "build": ", ".join(platform.python_build()),
+                                 "implementation":
+                                     platform.python_implementation(),
+                                 "version": platform.python_version(),
+                                 },
+                             "system": {
+                                 "info": platform.platform(),
+                                 "machine": platform.machine(),
+                                 "name": platform.system(),
+                                 "release": platform.release(),
+                                 "version": platform.version(),
+                                 },
+                             "tasks": {"background": redo_bg,
+                                       "segmentation": redo_seg
+                                       },
+                             },
+                             indent=2,
+                             sort_keys=True,
+                             cls=ExtendedJSONEncoder,
+                         ).split("\n"))
+
             # copy metadata/logs/tables from original file
             with h5py.File(self.job["path_in"]) as h5_src:
                 copy_metadata(h5_src=h5_src,
@@ -330,10 +434,20 @@ class DCNumJobRunner(threading.Thread):
                 mid_new = f"{mid_cur}_{mid_ap}" if mid_cur else mid_ap
                 hw.h5.attrs["experiment:run identifier"] = mid_new
 
+        trun = datetime.timedelta(seconds=round(time.monotonic() - time_start))
+        self.logger.info(f"Run duration: {str(trun)}")
+        self.logger.info(time.strftime("Run stop: %Y-%m-%d-%H.%M.%S",
+                                       time.gmtime()))
+        # Add the log file to the resulting .rtdc file
+        if self.path_log.exists():
+            with HDF5Writer(self.path_temp_out) as hw:
+                hw.store_log(
+                    f"dcnum-log-{time_string}",
+                    self.path_log.read_text().strip().split("\n"))
+
         # Rename the output file
         self.path_temp_out.rename(self.job["path_out"])
-        self._progress = 1.0
-        self._state = "done"
+        self.state = "done"
 
     def task_background(self):
         """Perform background computation task
@@ -357,7 +471,7 @@ class DCNumJobRunner(threading.Thread):
                 num_cpus=self.job["num_procs"],
                 # custom kwargs
                 **self.job["background_kwargs"]) as bic:
-
+            self._progress_bg = bic.image_proc
             bic.process()
         self.logger.info("Finished background computation")
 
@@ -365,14 +479,13 @@ class DCNumJobRunner(threading.Thread):
         self.logger.info("Starting segmentation and feature extraction")
         # Start writer thread
         writer_dq = collections.deque()
-        ds_kwds = dict(hdf5plugin.Zstd(clevel=5))
-        ds_kwds["fletcher32"] = True
+        ds_kwds = set_default_filter_kwargs()
         thr_write = DequeWriterThread(
             path_out=self.path_temp_out,
             dq=writer_dq,
             mode="w",
             ds_kwds=ds_kwds,
-            )
+        )
         thr_write.start()
 
         # Start segmentation thread
@@ -385,13 +498,24 @@ class DCNumJobRunner(threading.Thread):
         if self.job["debug"]:
             num_slots = 1
             num_extractors = 1
+            num_segmenters = 1
         elif seg_cls.hardware_processor == "cpu":  # CPU segmenter
+            # We could in principle set the number of slots to one and
+            # jave both number of extractors and number of segmenters set
+            # to the total number of CPUs. However, we would need more RAM
+            # (for caching the image data) and we also have more overhead.
+            # Having two slots shared between all workers is more efficient.
             num_slots = 2
+            # Split segmentation and feature extraction workers evenly.
             num_extractors = self.job["num_procs"] // 2
+            num_segmenters = self.job["num_procs"] - num_extractors
         else:  # GPU segmenter
             num_slots = 3
             num_extractors = self.job["num_procs"]
+            num_segmenters = 1
         num_extractors = max(1, num_extractors)
+        num_segmenters = max(1, num_segmenters)
+        self.job["segmenter_kwargs"]["num_workers"] = num_segmenters
 
         slot_chunks = mp_spawn.Array("i", num_slots)
         slot_states = mp_spawn.Array("u", num_slots)
@@ -410,7 +534,9 @@ class DCNumJobRunner(threading.Thread):
         fe_kwargs = QueueEventExtractor.get_init_kwargs(
             data=self.dtin,
             gate=gate.Gate(self.dtin, **self.job["gate_kwargs"]),
-            log_queue=self.log_queue)
+            log_queue=self.log_queue,
+            log_level=logging.DEBUG if self.job["debug"] else logging.INFO,
+            )
         fe_kwargs["extract_kwargs"] = self.job["feature_kwargs"]
 
         thr_feat = EventExtractorManagerThread(
@@ -437,16 +563,12 @@ class DCNumJobRunner(threading.Thread):
 
         # So in principle we are done here. We do not have to do anything
         # besides monitoring the progress.
-        pmin = 0.1  # from background computation
-        pmax = 0.95  # 5% reserved for cleanup
         while True:
             counted_frames = thr_coll.written_frames
             self.event_count = thr_coll.written_events
             td = time.monotonic() - t0
             # set the current status
-            self._progress = round(
-                pmin + counted_frames / data_size * (pmax - pmin),
-                3)
+            self._progress_ex = counted_frames / data_size
             self._segm_rate = counted_frames / (td or 0.03)
             time.sleep(.5)
             if counted_frames == data_size:
@@ -510,9 +632,9 @@ def join_thread_helper(thr, timeout, retries, logger, name):
         if thr.is_alive():
             logger.info(f"Waiting for '{name}' ({thr}")
         else:
-            logger.info(f"Joined thread '{name}'")
+            logger.debug(f"Joined thread '{name}'")
             break
     else:
         logger.error(f"Failed to join thread '{name}'")
-        raise ValueError(
-            f"Thread '{name}' ({thr}) did not join within {timeout*retries}s!")
+        raise ValueError(f"Thread '{name}' ({thr}) did not join"
+                         f"within {timeout * retries}s!")

dcnum/logic/json_encoder.py

@@ -0,0 +1,17 @@
+import json
+import numbers
+import pathlib
+
+import numpy as np
+
+
+class ExtendedJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, pathlib.Path):
+            return str(obj)
+        elif isinstance(obj, numbers.Integral):
+            return int(obj)
+        elif isinstance(obj, np.bool_):
+            return bool(obj)
+        # Let the base class default method raise the TypeError
+        return json.JSONEncoder.default(self, obj)

dcnum/read/cache.py

@@ -11,7 +11,7 @@ class HDF5ImageCache:
     def __init__(self,
                  h5ds: h5py.Dataset,
                  chunk_size: int = 1000,
-                 cache_size: int = 5,
+                 cache_size: int = 2,
                  boolean: bool = False):
         """An HDF5 image cache
 

dcnum/read/hdf5_data.py

@@ -25,7 +25,7 @@ class HDF5Data:
                  basins: List[Dict[List[str] | str]] = None,
                  logs: Dict[List[str]] = None,
                  tables: Dict[np.ndarray] = None,
-                 image_cache_size: int = 5,
+                 image_cache_size: int = 2,
                  ):
         # Init is in __setstate__ so we can pickle this class
         # and use it for multiprocessing.

dcnum/segm/segmenter_gpu.py

@@ -15,6 +15,7 @@ class GPUSegmenter(Segmenter, abc.ABC):
 
     def __init__(self,
                  *,
+                 num_workers: int = None,
                  kwargs_mask: Dict = None,
                  debug: bool = False,
                  **kwargs
@@ -31,6 +32,9 @@ class GPUSegmenter(Segmenter, abc.ABC):
             Additional, optional keyword arguments for `segment_approach`
             defined in the subclass.
         """
+        if num_workers not in [None, 1]:
+            raise ValueError(f"Number of workers must not be larger than 1 "
+                             f"for GPU segmenter, got '{num_workers}'!")
         super(GPUSegmenter, self).__init__(kwargs_mask=kwargs_mask,
                                            debug=debug,
                                            **kwargs)

dcnum/segm/segmenter_manager_thread.py

@@ -77,9 +77,9 @@ class SegmenterManagerThread(threading.Thread):
         self.debug = debug
 
     def run(self):
+        num_slots = len(self.slot_states)
         # We iterate over all the chunks of the image data.
         for chunk in self.image_data.iter_chunks():
-            num_slots = len(self.slot_states)
             cur_slot = 0
             empty_slots = 0
             # Wait for a free slot to perform segmentation (compute labels)
@@ -89,8 +89,11 @@ class SegmenterManagerThread(threading.Thread):
                 # - "s" the extractor processed the data and is waiting
                 #   for the segmenter
                 if self.slot_states[cur_slot] != "e":
+                    # It's the segmenters turn. Note that we use '!= "e"',
+                    # because the initial value is "\x00".
                     break
                 else:
+                    # Try another slot.
                     empty_slots += 1
                     cur_slot = (cur_slot + 1) % num_slots
                 if empty_slots >= num_slots:

dcnum/write/__init__.py

@@ -1,4 +1,5 @@
 # flake8: noqa: F401
 from .deque_writer_thread import DequeWriterThread
 from .queue_collector_thread import EventStash, QueueCollectorThread
-from .writer import HDF5Writer, copy_metadata, create_with_basins
+from .writer import (
+    HDF5Writer, copy_metadata, create_with_basins, set_default_filter_kwargs)

dcnum/write/queue_collector_thread.py

@@ -184,7 +184,7 @@ class QueueCollectorThread(threading.Thread):
 
             if len(cur_nevents) == 0:
                 self.logger.info(
-                    "Reached the end of the current dataset (frame "
+                    "Reached dataset end (frame "
                     # `last_idx` is the size of the dataset in the end,
                     # because `len(cur_nevents)` is always added to it.
                     f"{last_idx} of {len(self.feat_nevents)}).")
@@ -273,3 +273,6 @@ class QueueCollectorThread(threading.Thread):
 
             # Increment current frame index.
             last_idx += len(cur_nevents)
+
+        self.logger.info(f"Counted {self.written_events} events.")
+        self.logger.debug(f"Counted {self.written_frames} frames.")

dcnum/write/writer.py

@@ -21,11 +21,7 @@ class HDF5Writer:
         """Write deformability cytometry HDF5 data"""
         self.h5 = h5py.File(path, mode=mode, libver="latest")
         self.events = self.h5.require_group("events")
-        if ds_kwds is None:
-            ds_kwds = {}
-        for key, val in dict(hdf5plugin.Zstd(clevel=5)).items():
-            ds_kwds.setdefault(key, val)
-        ds_kwds.setdefault("fletcher32", True)
+        ds_kwds = set_default_filter_kwargs(ds_kwds)
         self.ds_kwds = ds_kwds
 
     def __enter__(self):
@@ -38,25 +34,24 @@ class HDF5Writer:
         self.h5.close()
 
     @staticmethod
-    def get_best_nd_chunks(item_shape):
+    def get_best_nd_chunks(item_shape, feat_dtype=np.float64):
         """Return best chunks for image data
 
         Chunking has performance implications. It’s recommended to keep the
-        total size of your chunks between 10 KiB and 1 MiB. This number defines
-        the maximum chunk size as well as half the maximum cache size for each
-        dataset.
+        total size of dataset chunks between 10 KiB and 1 MiB. This number
+        defines the maximum chunk size as well as half the maximum cache
+        size for each dataset.
         """
-        num_bytes = 1024**2  # between 10KiB and 1 MiB
-        if len(item_shape) == 0:
-            # scalar feature
-            chunk_size_int = 10000
-        else:
-            event_size = np.prod(item_shape) * np.dtype(np.uint8).itemsize
-            chunk_size = num_bytes / event_size
-            chunk_size_int = max(1, int(np.floor(chunk_size)))
+        # set image feature chunk size to approximately 1MiB
+        num_bytes = 1024 ** 2
+        event_size = np.prod(item_shape) * np.dtype(feat_dtype).itemsize
+        chunk_size = num_bytes / event_size
+        # Set minimum chunk size to 10 so that we can have at least some
+        # compression performance.
+        chunk_size_int = max(10, int(np.floor(chunk_size)))
         return tuple([chunk_size_int] + list(item_shape))
 
-    def require_feature(self, feat, item_shape, dtype, ds_kwds=None):
+    def require_feature(self, feat, item_shape, feat_dtype, ds_kwds=None):
         """Create a new feature in the "events" group"""
 
         if ds_kwds is None:
@@ -67,9 +62,10 @@ class HDF5Writer:
             dset = self.events.create_dataset(
                 feat,
                 shape=tuple([0] + list(item_shape)),
-                dtype=dtype,
+                dtype=feat_dtype,
                 maxshape=tuple([None] + list(item_shape)),
-                chunks=self.get_best_nd_chunks(item_shape),
+                chunks=self.get_best_nd_chunks(item_shape,
+                                               feat_dtype=feat_dtype),
                 **ds_kwds)
             if len(item_shape) == 2:
                 dset.attrs.create('CLASS', np.string_('IMAGE'))
@@ -137,7 +133,7 @@ class HDF5Writer:
             data = 255 * np.array(data, dtype=np.uint8)
         ds, offset = self.require_feature(feat=feat,
                                           item_shape=data.shape[1:],
-                                          dtype=data.dtype)
+                                          feat_dtype=data.dtype)
         dsize = data.shape[0]
         ds.resize(offset + dsize, axis=0)
         ds[offset:offset + dsize] = data
@@ -249,10 +245,7 @@ def copy_metadata(h5_src: h5py.File,
     are not defined already are added.
     """
     # compress data
-    ds_kwds = {}
-    for key, val in dict(hdf5plugin.Zstd(clevel=5)).items():
-        ds_kwds.setdefault(key, val)
-    ds_kwds.setdefault("fletcher32", True)
+    ds_kwds = set_default_filter_kwargs()
     # set attributes
     src_attrs = dict(h5_src.attrs)
     for kk in src_attrs:
@@ -267,18 +260,31 @@ def copy_metadata(h5_src: h5py.File,
                 h5_dst.require_group(topic)
                 if key not in h5_dst[topic]:
                     data = h5_src[topic][key][:]
-                    if data.dtype == np.dtype("O"):
-                        # convert variable-length strings to fixed-length
-                        max_length = max([len(line) for line in data])
-                        data = np.asarray(data, dtype=f"S{max_length}")
-                    ds = h5_dst[topic].create_dataset(
-                        name=key,
-                        data=data,
-                        **ds_kwds
-                    )
-                    # help with debugging and add some meta-metadata
-                    ds.attrs.update(h5_src[topic][key].attrs)
-                    soft_strings = [ds.attrs.get("software"),
-                                    f"dcnum {version}"]
-                    soft_strings = [s for s in soft_strings if s is not None]
-                    ds.attrs["software"] = " | ".join(soft_strings)
+                    if data.size:  # ignore empty datasets
+                        if data.dtype == np.dtype("O"):
+                            # convert variable-length strings to fixed-length
+                            max_length = max([len(line) for line in data])
+                            data = np.asarray(data, dtype=f"S{max_length}")
+                        ds = h5_dst[topic].create_dataset(
+                            name=key,
+                            data=data,
+                            **ds_kwds
+                        )
+                        # help with debugging and add some meta-metadata
+                        ds.attrs.update(h5_src[topic][key].attrs)
+                        soft_strgs = [ds.attrs.get("software"),
+                                      f"dcnum {version}"]
+                        soft_strgs = [s for s in soft_strgs if s is not None]
+                        ds.attrs["software"] = " | ".join(soft_strgs)
+
+
+def set_default_filter_kwargs(ds_kwds=None, compression=True):
+    if ds_kwds is None:
+        ds_kwds = {}
+    if compression:
+        # compression
+        for key, val in dict(hdf5plugin.Zstd(clevel=5)).items():
+            ds_kwds.setdefault(key, val)
+    # checksums
+    ds_kwds.setdefault("fletcher32", True)
+    return ds_kwds

{dcnum-0.16.1.dist-info → dcnum-0.16.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.16.1
+Version: 0.16.3
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.16.1.dist-info → dcnum-0.16.3.dist-info}/RECORD

@@ -1,14 +1,14 @@
 dcnum/__init__.py,sha256=hcawIKS7utYiOyVhOAX9t7K3xYzP1b9862VV0b6qSrQ,74
-dcnum/_version.py,sha256=djn14s8hpDNt1rX_1-rosukF6WH__dfZ71HDUa63720,413
+dcnum/_version.py,sha256=KgYmvPTSKj3FMTlG8S1wtqb14ST1EcsHHNSwLJp3foQ,413
 dcnum/feat/__init__.py,sha256=JqlgzOgDJhoTk8WVYcIiKTWq9EAM16_jGivzOtN6JGo,325
-dcnum/feat/event_extractor_manager_thread.py,sha256=V2idRAlC7bdsA8I40RAUkqz3jtWmTeb4cjPXpRjr8Ik,6145
+dcnum/feat/event_extractor_manager_thread.py,sha256=ypsGEwmM_ohHCnnl8g1vpruezFAkH0drIU1AOngH5Bg,6837
 dcnum/feat/gate.py,sha256=srobj5p2RDr_S2SUtbwGbTKatnc_aPSndt0cR2P9zoY,7060
-dcnum/feat/queue_event_extractor.py,sha256=RYz0VNtV8OVGDFn9MrYWM5NaB4rXLiJch9MTKDnccs0,14453
+dcnum/feat/queue_event_extractor.py,sha256=o7K4p5VNExnaO6lgnlHrVk_qPhbXzocUyFhUcoP7OAU,14970
 dcnum/feat/feat_background/__init__.py,sha256=OTmMuazHNaSrZb2XW4cnJ6PlgJLbKrPbaidpEixYa0A,341
-dcnum/feat/feat_background/base.py,sha256=N1SL5NCZ7gTS5AQONxEH31PFJBx0zvVjCaA4mprheuY,7974
+dcnum/feat/feat_background/base.py,sha256=KA1H5giTyMBADex2-LmGbu7B1PEAKjiCUAvSF89WiZs,8375
 dcnum/feat/feat_background/bg_copy.py,sha256=aHabgizRuwIdOH8S850Cun9NsmpMzo4B3yHWv1aFNFI,645
-dcnum/feat/feat_background/bg_roll_median.py,sha256=3wbY_zoNx7zSs8ZgORQBXw0dfD_3Xai4h7-RyPETnoI,13048
-dcnum/feat/feat_background/bg_sparse_median.py,sha256=JzrAAOKOXJS1gI4y3RPTvDeNPUa2iliXZqlwADp4Edc,17840
+dcnum/feat/feat_background/bg_roll_median.py,sha256=FfC3v1cX8mreLO971C_kTpFRBtuJP4Sv-Hj1Wj8yb3Q,12826
+dcnum/feat/feat_background/bg_sparse_median.py,sha256=CDO8X7-7agBxTrC79lskt0zWTaSex6ouxUVfxImhgs4,17630
 dcnum/feat/feat_brightness/__init__.py,sha256=o6AebVlmydwNgVF5kW6ITqJyFreoKrU3Ki_3EC8If-s,155
 dcnum/feat/feat_brightness/bright_all.py,sha256=Z5b-xkw7g7ejMpbGmdUqrxGRymqFhAQsZ938gaGXk9Y,3102
 dcnum/feat/feat_brightness/common.py,sha256=JX49EszYDmnvoOKXFVV1CalEIWRmOuY5EryNbqGbdac,156
@@ -19,26 +19,27 @@ dcnum/feat/feat_texture/__init__.py,sha256=6StM9S540UVtdFFR3bHa7nfCTomeVdoo7Uy9C
 dcnum/feat/feat_texture/common.py,sha256=COXHpXS-7DMouGu3WF83I76L02Sr7P9re4lxajh6g0E,439
 dcnum/feat/feat_texture/tex_all.py,sha256=eGjjNfPpfZw7FA_VNFCIMiU38KD0qcGbxLciYy-tCiA,4097
 dcnum/logic/__init__.py,sha256=5hgAQMp2YGsqpWoeTQ9qxGAWfxPOKQjJsYyNsS49t0g,131
-dcnum/logic/ctrl.py,sha256=8yPhIOB9Pju2D8s65OlEDxXX1fu4iqHvo7MpSqUe7z8,21381
+dcnum/logic/ctrl.py,sha256=aqXCH_yyrfifeAxmpW6Cg-FQIwTBjpElHbva60ghYpY,26655
 dcnum/logic/job.py,sha256=M0Q-Rfcm-zkTXTQc79W6YSNUjUlgmRPG0Ikbdn1aOpY,4608
+dcnum/logic/json_encoder.py,sha256=dy44ArmdnxpUfxxONmKdIv-fde3aTXPjZDN0HPATaxs,467
 dcnum/meta/__init__.py,sha256=cQT_HN5yDKzMnZM8CUyNmeA68OhE3ENO_rvFmgDj95c,40
 dcnum/meta/ppid.py,sha256=_xUqJal4wBqgic2aRN3ZMMteTggHeYGs44nrYbTKlpQ,8107
 dcnum/read/__init__.py,sha256=iV2wrBMdwJgpXaphNiiAVybndDzTTv0CAGRNXyvxcLY,157
-dcnum/read/cache.py,sha256=mr2DBJZYgNIAiz64TQ4cgkPmRt8nJWBvgkOpaz-p6Yg,5467
+dcnum/read/cache.py,sha256=0tMurtHOA7VnPNpfeAGi-dxWXfYhL5wmWuXb6ka_eEo,5467
 dcnum/read/const.py,sha256=SVlvEJiRIHyTyUlWG24_ogcnT5nTxCi0CRslNuNP56I,282
-dcnum/read/hdf5_data.py,sha256=jUPoXgn52eJZrF4uOpR2-fBLaQX9Ezw3tcuAlI5LnF8,18817
+dcnum/read/hdf5_data.py,sha256=8g39CZoFIa2tUvizZt_vzMeoCUcTkkt3AkXK6MMN0iY,18817
 dcnum/segm/__init__.py,sha256=iiq_1A9DU5wMUcKnsZ53E7NyzCkbZCJeUDimzunE-OM,247
 dcnum/segm/segm_thresh.py,sha256=aLVTydPjbrgKDkZFY3Ew5CX-miwOw71meHfxcO5EjCc,1176
 dcnum/segm/segmenter.py,sha256=F3gCp-Z51F9GxdFYPF1CHjnbfgqnS0_g-34lJF2tMCM,10611
 dcnum/segm/segmenter_cpu.py,sha256=tCY105rVr9_0RIq2618qnF1ueHRj7UtuK_nUBoAg-nY,10743
-dcnum/segm/segmenter_gpu.py,sha256=aIBieTjcr4YaOHUhYiOOh_EpKUzQLBhanpBrtPSUL5k,1900
-dcnum/segm/segmenter_manager_thread.py,sha256=xtuk7gnk7xhoRoV_J97rrv7IR3JgeRvVewCDT-chqpk,5172
-dcnum/write/__init__.py,sha256=6vAQECatcd7DJMXFEuab1wdvEiaxisbY8_qmK5tzIwY,207
+dcnum/segm/segmenter_gpu.py,sha256=tL2X5BN0jKmhC7wgfG0hygd-6UpG1ZCVuKe5OP1qde0,2133
+dcnum/segm/segmenter_manager_thread.py,sha256=2znDaKedSueomcU1pbHtFmVcGoHzp--sf494VgJF_Tk,5342
+dcnum/write/__init__.py,sha256=Cpn3LqL18hh8OScUnGp_AnNfpWPpKW-oAJZH6ot7aRA,241
 dcnum/write/deque_writer_thread.py,sha256=R4x3p-HZUls3upCBX3vV1VqSdSmaiHdrAswMJj_tVpk,1643
-dcnum/write/queue_collector_thread.py,sha256=BivSe5ZA-rTPH4sridXU1yFB6CP7LYzIFudLMbN481s,11793
-dcnum/write/writer.py,sha256=8DB4O14tXFisiMDdHawHBdQHOg_uXZkRFbgXNdWdCHQ,10167
-dcnum-0.16.1.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
-dcnum-0.16.1.dist-info/METADATA,sha256=4tTCYJ4d49W7OmwvhDriK295uKuPzCMgecWvUJvcksw,2172
-dcnum-0.16.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-dcnum-0.16.1.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
-dcnum-0.16.1.dist-info/RECORD,,
+dcnum/write/queue_collector_thread.py,sha256=c0Z6uZfZ3B8xsTMCB5jglEukM5sesA9HgEawBk_YEUA,11910
+dcnum/write/writer.py,sha256=Hr37OSDJGUpJJ4OufJHYYBanE26GiNwUPOMAt-5Yc2Y,10478
+dcnum-0.16.3.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
+dcnum-0.16.3.dist-info/METADATA,sha256=BChO0SWVq5w9ZEFvvP0KDsFd-T1WRbMZkhW2xCBeVWc,2172
+dcnum-0.16.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+dcnum-0.16.3.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
+dcnum-0.16.3.dist-info/RECORD,,