dcnum 0.18.0__py3-none-any.whl → 0.19.1__py3-none-any.whl

dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.18.0'
-__version_tuple__ = version_tuple = (0, 18, 0)
+__version__ = version = '0.19.1'
+__version_tuple__ = version_tuple = (0, 19, 1)

dcnum/feat/feat_background/bg_roll_median.py

@@ -119,7 +119,7 @@ class BackgroundRollMed(Background):
         """Check user-defined properties of this class
 
         This method primarily exists so that the CLI knows which
-        keyword arguements can be passed to this class.
+        keyword arguments can be passed to this class.
 
         Parameters
         ----------
@@ -132,7 +132,8 @@ class BackgroundRollMed(Background):
             `kernel_size` will not increase computation speed. Larger
             values lead to a higher memory consumption.
         """
-        assert kernel_size > 0
+        assert kernel_size > 0, "kernel size must be positive number"
+        assert kernel_size % 2 == 0, "kernel size must be even number"
         assert batch_size > kernel_size
 
     def get_slices_for_batch(self, batch_index=0):

dcnum/logic/ctrl.py

@@ -1,11 +1,9 @@
 import collections
 import datetime
-import hashlib
 import json
 import logging
 from logging.handlers import QueueListener
 import multiprocessing as mp
-import numbers
 import os
 import pathlib
 import platform
@@ -16,7 +14,6 @@ import traceback
 import uuid
 
 import h5py
-import numpy as np
 
 from ..feat.feat_background.base import get_available_background_methods
 from ..feat.queue_event_extractor import QueueEventExtractor
@@ -313,7 +310,17 @@ class DCNumJobRunner(threading.Thread):
             # Whether pipeline hash is invalid.
             ppid.compute_pipeline_hash(**datdict) != dathash
             # Whether the input file is the original output of the pipeline.
-            or len(self.draw) != evyield)
+            or len(self.draw) != evyield
+            # If index mapping is defined, then we always redo the pipeline.
+            # If the pipeline hashes are identical and index mapping is not
+            # None, then both pipelines were done with index mapping.
+            # But applying the same pipeline with index mapping in series
+            # will lead to a different result in the second run (e.g. 1st
+            # pipeline run: take every 2nd event; 2nd pipeline run: take
+            # every second event -> results in every 4th event in output of
+            # second pipeline run).
+            or self.draw.index_mapping is not None
+        )
         # Do we have to recompute the background data? In addition to the
         # hash sanity check above, check the generation, input data,
         # and background pipeline identifiers.
@@ -387,21 +394,7 @@ class DCNumJobRunner(threading.Thread):
             hw.h5.attrs["pipeline:dcnum yield"] = self.event_count
             # index mapping information
             im = self.job.kwargs["data_kwargs"].get("index_mapping", None)
-            if im is None:
-                dim = "0"
-            elif isinstance(im, numbers.Number):
-                dim = f"{im}"
-            elif isinstance(im, slice):
-                dim = (f"{im.start if im.start is not None else 'n'}"
-                       + f"-{im.stop if im.stop is not None else 'n'}"
-                       + f"-{im.step if im.step is not None else 'n'}"
-                       )
-            elif isinstance(im, (list, np.ndarray)):
-                idhash = hashlib.md5(
-                    np.array(im, dtype=np.uint32).tobytes()).hexdigest()
-                dim = f"h-{idhash[:8]}"
-            else:
-                dim = "unknown"
+            dim = HDF5Data.get_ppid_index_mapping(im)
             hw.h5.attrs["pipeline:dcnum mapping"] = dim
             # regular metadata
             hw.h5.attrs["experiment:event count"] = self.event_count

dcnum/read/hdf5_data.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import hashlib
 import io
 import json
+import numbers
 import pathlib
 import tempfile
 from typing import Dict, BinaryIO, List
@@ -293,7 +295,9 @@ class HDF5Data:
         self.h5.close()
 
     def get_ppid(self):
-        return self.get_ppid_from_ppkw({"pixel_size": self.pixel_size})
+        return self.get_ppid_from_ppkw(
+            {"pixel_size": self.pixel_size,
+             "index_mapping": self.index_mapping})
 
     @classmethod
     def get_ppid_code(cls):
@@ -304,10 +308,34 @@ class HDF5Data:
         # Data does not really fit into the PPID scheme we use for the rest
         # of the pipeline. This implementation here is custom.
         code = cls.get_ppid_code()
+        # pixel size
         ppid_ps = f"{kwargs['pixel_size']:.8f}".rstrip("0")
-        kwid = "^".join([f"p={ppid_ps}"])
+        # index mapping
+        ppid_im = cls.get_ppid_index_mapping(kwargs.get("index_mapping", None))
+        kwid = "^".join([f"p={ppid_ps}", f"i={ppid_im}"])
         return ":".join([code, kwid])
 
+    @staticmethod
+    def get_ppid_index_mapping(index_mapping):
+        """Return the pipeline identifier part for index mapping"""
+        im = index_mapping
+        if im is None:
+            dim = "0"
+        elif isinstance(im, numbers.Integral):
+            dim = f"{im}"
+        elif isinstance(im, slice):
+            dim = (f"{im.start if im.start is not None else 'n'}"
+                   + f"-{im.stop if im.stop is not None else 'n'}"
+                   + f"-{im.step if im.step is not None else 'n'}"
+                   )
+        elif isinstance(im, (list, np.ndarray)):
+            idhash = hashlib.md5(
+                np.array(im, dtype=np.uint32).tobytes()).hexdigest()
+            dim = f"h-{idhash[:8]}"
+        else:
+            dim = "unknown"
+        return dim
+
     @staticmethod
     def get_ppkw_from_ppid(dat_ppid):
         # Data does not fit in the PPID scheme we use, but we still
@@ -321,6 +349,20 @@ class HDF5Data:
             var, val = item.split("=")
             if var == "p":
                 kwargs["pixel_size"] = float(val)
+            elif var == "i":
+                if val.startswith("h-") or val == "unknown":
+                    raise ValueError(f"Cannot invert index mapping {val}")
+                elif val == "0":
+                    kwargs["index_mapping"] = None
+                elif val.count("-"):
+                    start, stop, step = val.split("-")
+                    kwargs["index_mapping"] = slice(
+                        None if start == "n" else int(start),
+                        None if stop == "n" else int(stop),
+                        None if step == "n" else int(step)
+                    )
+                else:
+                    kwargs["index_mapping"] = int(val)
             else:
                 raise ValueError(f"Invalid parameter '{var}'!")
         return kwargs

dcnum/read/mapped.py

@@ -54,12 +54,10 @@ def _get_mapping_indices_cached(
         return np.arange(index_mapping)
     elif isinstance(index_mapping, tuple):
         im_slice = slice(*index_mapping)
-        if im_slice.step is not None:
-            raise NotImplementedError("Slices with step not implemented yet")
         if im_slice.stop is None or im_slice.start is None:
             raise NotImplementedError(
                 "Slices must have start and stop defined")
-        return np.arange(im_slice.start, im_slice.stop)
+        return np.arange(im_slice.start, im_slice.stop, im_slice.step)
     elif isinstance(index_mapping, list):
         return np.array(index_mapping, dtype=np.uint32)
     else:

{dcnum-0.18.0.dist-info → dcnum-0.19.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.18.0
+Version: 0.19.1
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.18.0.dist-info → dcnum-0.19.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dcnum/__init__.py,sha256=hcawIKS7utYiOyVhOAX9t7K3xYzP1b9862VV0b6qSrQ,74
-dcnum/_version.py,sha256=qQfZIFMnYUfFvdZIkQ7KTINCu5N5FVIhIaHGQZyIg1E,413
+dcnum/_version.py,sha256=wOLHPF5OO0ubEMjzvaXg4CVhVL1uy5Ci-sH1WTZH0Dg,413
 dcnum/feat/__init__.py,sha256=jUJYWTD3VIoDNKrmryXbjHb1rGwYtK4b7VPWihYgUoo,325
 dcnum/feat/event_extractor_manager_thread.py,sha256=Ocid_t1awH6pOmurCmKYkC51XsXB0-DoN3fzjFDgE4c,7129
 dcnum/feat/gate.py,sha256=svbObmqpYdqPawpfrsEjTiUPJXf24GrNi8PXTKT-z44,7225
@@ -7,7 +7,7 @@ dcnum/feat/queue_event_extractor.py,sha256=XhA930QVQ1Z4saisbcGTrEut-fSgwTyfDn6b9
 dcnum/feat/feat_background/__init__.py,sha256=OTmMuazHNaSrZb2XW4cnJ6PlgJLbKrPbaidpEixYa0A,341
 dcnum/feat/feat_background/base.py,sha256=IYBFfsGXBfmFnZfD9QrmfrXbJtFSfVOS-v-u-uxSThs,7985
 dcnum/feat/feat_background/bg_copy.py,sha256=muU-6eTUf3HTA2K2asrLWcR_hbRMjdygZROmjNXCm7Q,923
-dcnum/feat/feat_background/bg_roll_median.py,sha256=G1atu6R8FWDe-Lblr1KcdP368Xx-kj6oy3q2mhljxB0,13064
+dcnum/feat/feat_background/bg_roll_median.py,sha256=j3seExcWkk6IeFIOO4zkj-LIA7ryig9bmGYYj_dDgoM,13173
 dcnum/feat/feat_background/bg_sparse_median.py,sha256=-CShs4UAuZd00rACaXTZj3dccKevhcSGRsILFgMPLWo,20705
 dcnum/feat/feat_brightness/__init__.py,sha256=o6AebVlmydwNgVF5kW6ITqJyFreoKrU3Ki_3EC8If-s,155
 dcnum/feat/feat_brightness/bright_all.py,sha256=vf8xaYBdKD24hHUXdkI0_S7nbr7m49KW6gvuWvbHDVg,4545
@@ -20,7 +20,7 @@ dcnum/feat/feat_texture/__init__.py,sha256=6StM9S540UVtdFFR3bHa7nfCTomeVdoo7Uy9C
 dcnum/feat/feat_texture/common.py,sha256=COXHpXS-7DMouGu3WF83I76L02Sr7P9re4lxajh6g0E,439
 dcnum/feat/feat_texture/tex_all.py,sha256=eGjjNfPpfZw7FA_VNFCIMiU38KD0qcGbxLciYy-tCiA,4097
 dcnum/logic/__init__.py,sha256=7J3GrwJInNQbrLk61HRIV7X7p69TAIbMYpR34hh6u14,177
-dcnum/logic/ctrl.py,sha256=dZlQBUhFrBOs5jkdtFX60g_1M_3A05kXqmCkWTTfyHk,27998
+dcnum/logic/ctrl.py,sha256=FvVXbrP7WqgYeDznep0KyfMck3cbCO8Yoli8P6clRPc,27956
 dcnum/logic/job.py,sha256=M0Q-Rfcm-zkTXTQc79W6YSNUjUlgmRPG0Ikbdn1aOpY,4608
 dcnum/logic/json_encoder.py,sha256=cxMnqisbKEVf-rVcw6rK2BBAb6iz_hKFaGl81kK36lQ,571
 dcnum/meta/__init__.py,sha256=AVqRgyKXO1orKnE305h88IBvoZ1oz6X11HN1WP5nGvg,60
@@ -29,8 +29,8 @@ dcnum/meta/ppid.py,sha256=Q3jg8lZt5tlGIby_-7rBqTANesMjJrmxASXZhsvBD_Y,7706
 dcnum/read/__init__.py,sha256=8uGj4YN7pDP4FO9TkZWXrpScwTLVWSEZexFq-TS9vsA,215
 dcnum/read/cache.py,sha256=kC2Y9hXA92ARQ2Vgm1kBFCU-s6TPE1tPYvpzWI0aPow,5619
 dcnum/read/const.py,sha256=8ih8rlWM7ntp8phrr9dh22hXXb210igSCatOSI9Ou30,463
-dcnum/read/hdf5_data.py,sha256=sGqwaj3HpceSsctlVFty-3TRnWeN09WNfUh30xkt5b0,20598
-dcnum/read/mapped.py,sha256=X__uqz5rT-N_xJhr6QsmsvXzIqu806piuTyjTHd_Ewg,2777
+dcnum/read/hdf5_data.py,sha256=psMN2CGorU4uFO1nlGcpUxKFLZ9HaKCReTi7tVx50tg,22291
+dcnum/read/mapped.py,sha256=Oh1jH2yVqWBPomEf8vlGvsGOMc02ldapAAjgNo-bS7g,2676
 dcnum/segm/__init__.py,sha256=iiq_1A9DU5wMUcKnsZ53E7NyzCkbZCJeUDimzunE-OM,247
 dcnum/segm/segm_thresh.py,sha256=lMf-lso_O_5Q5lJiiIQdYkM3zlj4uwNz9cNvLxVMeXc,1396
 dcnum/segm/segmenter.py,sha256=gVzmP6CuwI9Qfk8GN_xWGu_xbtVTOhxIOWn-2yr_H1Y,12220
@@ -41,8 +41,8 @@ dcnum/write/__init__.py,sha256=Cpn3LqL18hh8OScUnGp_AnNfpWPpKW-oAJZH6ot7aRA,241
 dcnum/write/deque_writer_thread.py,sha256=KpJ6po8JPlM696MITN-bhNnWQcy9E-qlhg9g-uzoPZg,1710
 dcnum/write/queue_collector_thread.py,sha256=YQ6pvKNmCDf1C6HVx6gOA-q-FBoI6nkhOo-tAVYnyag,11906
 dcnum/write/writer.py,sha256=nlJfQCPoW2Wze72y_256G4qmgYMdh5mL0vpvqg7lSaU,11728
-dcnum-0.18.0.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
-dcnum-0.18.0.dist-info/METADATA,sha256=Vnx_UzKsj2G9X16joXk6QOZWmDmvVmBgwXiuiGvK65o,2194
-dcnum-0.18.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-dcnum-0.18.0.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
-dcnum-0.18.0.dist-info/RECORD,,
+dcnum-0.19.1.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
+dcnum-0.19.1.dist-info/METADATA,sha256=tWMwpFt4Nn8vs1H0aRTBWDmH3pej-O3gWHJ6ESkbvSw,2194
+dcnum-0.19.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+dcnum-0.19.1.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
+dcnum-0.19.1.dist-info/RECORD,,