dcnum 0.21.3__tar.gz → 0.22.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dcnum might be problematic. Click here for more details.

Files changed (108) hide show
  1. {dcnum-0.21.3 → dcnum-0.22.0}/CHANGELOG +10 -1
  2. {dcnum-0.21.3/src/dcnum.egg-info → dcnum-0.22.0}/PKG-INFO +1 -1
  3. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/_version.py +2 -2
  4. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/event_extractor_manager_thread.py +1 -1
  5. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/logic/ctrl.py +1 -1
  6. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/meta/ppid.py +1 -1
  7. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/segm/__init__.py +2 -2
  8. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/segm/segm_thresh.py +4 -4
  9. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/segm/segmenter.py +64 -40
  10. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/segm/segmenter_manager_thread.py +4 -3
  11. dcnum-0.21.3/src/dcnum/segm/segmenter_cpu.py → dcnum-0.22.0/src/dcnum/segm/segmenter_mpo.py +111 -41
  12. dcnum-0.22.0/src/dcnum/segm/segmenter_sto.py +110 -0
  13. {dcnum-0.21.3 → dcnum-0.22.0/src/dcnum.egg-info}/PKG-INFO +1 -1
  14. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum.egg-info/SOURCES.txt +4 -2
  15. {dcnum-0.21.3 → dcnum-0.22.0}/tests/helper_methods.py +29 -0
  16. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_ppid_segm.py +1 -1
  17. dcnum-0.22.0/tests/test_segm_base.py +143 -0
  18. dcnum-0.21.3/tests/test_segm_base.py → dcnum-0.22.0/tests/test_segm_mpo.py +132 -167
  19. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_segm_no_mask_proc.py +2 -2
  20. dcnum-0.22.0/tests/test_segm_sto.py +294 -0
  21. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_segm_thresh.py +4 -4
  22. dcnum-0.21.3/src/dcnum/segm/segmenter_gpu.py +0 -63
  23. {dcnum-0.21.3 → dcnum-0.22.0}/.github/workflows/check.yml +0 -0
  24. {dcnum-0.21.3 → dcnum-0.22.0}/.github/workflows/deploy_pypi.yml +0 -0
  25. {dcnum-0.21.3 → dcnum-0.22.0}/.gitignore +0 -0
  26. {dcnum-0.21.3 → dcnum-0.22.0}/.readthedocs.yml +0 -0
  27. {dcnum-0.21.3 → dcnum-0.22.0}/LICENSE +0 -0
  28. {dcnum-0.21.3 → dcnum-0.22.0}/README.rst +0 -0
  29. {dcnum-0.21.3 → dcnum-0.22.0}/docs/conf.py +0 -0
  30. {dcnum-0.21.3 → dcnum-0.22.0}/docs/extensions/github_changelog.py +0 -0
  31. {dcnum-0.21.3 → dcnum-0.22.0}/docs/index.rst +0 -0
  32. {dcnum-0.21.3 → dcnum-0.22.0}/docs/requirements.txt +0 -0
  33. {dcnum-0.21.3 → dcnum-0.22.0}/pyproject.toml +0 -0
  34. {dcnum-0.21.3 → dcnum-0.22.0}/setup.cfg +0 -0
  35. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/__init__.py +0 -0
  36. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/__init__.py +0 -0
  37. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_background/__init__.py +0 -0
  38. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_background/base.py +0 -0
  39. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_background/bg_copy.py +0 -0
  40. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_background/bg_roll_median.py +0 -0
  41. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_background/bg_sparse_median.py +0 -0
  42. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_brightness/__init__.py +0 -0
  43. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_brightness/bright_all.py +0 -0
  44. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_brightness/common.py +0 -0
  45. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_contour/__init__.py +0 -0
  46. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_contour/contour.py +0 -0
  47. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_contour/moments.py +0 -0
  48. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_contour/volume.py +0 -0
  49. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_texture/__init__.py +0 -0
  50. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_texture/common.py +0 -0
  51. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_texture/tex_all.py +0 -0
  52. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/gate.py +0 -0
  53. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/queue_event_extractor.py +0 -0
  54. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/logic/__init__.py +0 -0
  55. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/logic/job.py +0 -0
  56. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/logic/json_encoder.py +0 -0
  57. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/meta/__init__.py +0 -0
  58. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/meta/paths.py +0 -0
  59. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/read/__init__.py +0 -0
  60. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/read/cache.py +0 -0
  61. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/read/const.py +0 -0
  62. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/read/hdf5_data.py +0 -0
  63. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/read/mapped.py +0 -0
  64. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/write/__init__.py +0 -0
  65. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/write/deque_writer_thread.py +0 -0
  66. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/write/queue_collector_thread.py +0 -0
  67. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/write/writer.py +0 -0
  68. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum.egg-info/dependency_links.txt +0 -0
  69. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum.egg-info/requires.txt +0 -0
  70. {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum.egg-info/top_level.txt +0 -0
  71. {dcnum-0.21.3 → dcnum-0.22.0}/tests/conftest.py +0 -0
  72. {dcnum-0.21.3 → dcnum-0.22.0}/tests/data/fmt-hdf5_cytoshot_extended-moments-features.zip +0 -0
  73. {dcnum-0.21.3 → dcnum-0.22.0}/tests/data/fmt-hdf5_cytoshot_full-features_2023.zip +0 -0
  74. {dcnum-0.21.3 → dcnum-0.22.0}/tests/data/fmt-hdf5_cytoshot_full-features_2024.zip +0 -0
  75. {dcnum-0.21.3 → dcnum-0.22.0}/tests/data/fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip +0 -0
  76. {dcnum-0.21.3 → dcnum-0.22.0}/tests/data/fmt-hdf5_shapein_empty.zip +0 -0
  77. {dcnum-0.21.3 → dcnum-0.22.0}/tests/data/fmt-hdf5_shapein_raw-with-variable-length-logs.zip +0 -0
  78. {dcnum-0.21.3 → dcnum-0.22.0}/tests/requirements.txt +0 -0
  79. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_background_base.py +0 -0
  80. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_background_bg_copy.py +0 -0
  81. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_background_bg_roll_median.py +0 -0
  82. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_background_bg_sparsemed.py +0 -0
  83. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_brightness.py +0 -0
  84. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_event_extractor_manager.py +0 -0
  85. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_gate.py +0 -0
  86. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_haralick.py +0 -0
  87. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_moments_based.py +0 -0
  88. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_moments_based_extended.py +0 -0
  89. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_volume.py +0 -0
  90. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_init.py +0 -0
  91. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_logic_job.py +0 -0
  92. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_logic_join.py +0 -0
  93. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_logic_json.py +0 -0
  94. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_logic_pipeline.py +0 -0
  95. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_paths.py +0 -0
  96. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_ppid_base.py +0 -0
  97. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_ppid_bg.py +0 -0
  98. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_ppid_data.py +0 -0
  99. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_ppid_feat.py +0 -0
  100. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_ppid_gate.py +0 -0
  101. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_read_basin.py +0 -0
  102. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_read_concat_hdf5.py +0 -0
  103. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_read_hdf5.py +0 -0
  104. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_read_hdf5_basins.py +0 -0
  105. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_read_hdf5_index_mapping.py +0 -0
  106. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_write_deque_writer_thread.py +0 -0
  107. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_write_queue_collector_thread.py +0 -0
  108. {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_write_writer.py +0 -0
@@ -1,3 +1,13 @@
1
+ 0.22.0
2
+ - fix: GPUSegmenter did not perform mask postprocessing
3
+ (this was not actually fixed in 0.21.0)
4
+ - tests: implement mock STO segmenter for testing (#3)
5
+ - tests: add tests for bg_off in segmentation
6
+ - ref: increment DCNUM_PPID_GENERATION to 10
7
+ - ref: renamed CPUSegmenter to MPOSegmenter
8
+ - ref: renamed GPUSegmenter to STOSegmenter
9
+ 0.21.4
10
+ - fix: division by zero error when computing stall time
1
11
  0.21.3
2
12
  - fix: negative stall time in log messages
3
13
  0.21.2
@@ -5,7 +15,6 @@
5
15
  0.21.1
6
16
  - enh: support boolean images in mask postprocessing
7
17
  0.21.0
8
- - fix: GPUSegmenter did not perform mask postprocessing
9
18
  - ref: enable mask postprocessing by default for all segmenters
10
19
  - ref: increment DCNUM_PPID_GENERATION to 9
11
20
  0.20.4
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcnum
3
- Version: 0.21.3
3
+ Version: 0.22.0
4
4
  Summary: numerics toolbox for imaging deformability cytometry
5
5
  Author: Maximilian Schlögel, Paul Müller
6
6
  Maintainer-email: Paul Müller <dev@craban.de>
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '0.21.3'
16
- __version_tuple__ = version_tuple = (0, 21, 3)
15
+ __version__ = version = '0.22.0'
16
+ __version_tuple__ = version_tuple = (0, 22, 0)
@@ -98,7 +98,7 @@ class EventExtractorManagerThread(threading.Thread):
98
98
  if (ldq := len(self.writer_dq)) > 1000:
99
99
  time.sleep(1)
100
100
  ldq2 = len(self.writer_dq)
101
- stall_time = max(0., (ldq2 - 200) / (ldq - ldq2))
101
+ stall_time = max(0., (ldq2 - 200) / ((ldq - ldq2) or 1))
102
102
  time.sleep(stall_time)
103
103
  self.logger.warning(
104
104
  f"Stalled {stall_time + 1:.1f}s for slow writer "
@@ -648,7 +648,7 @@ class DCNumJobRunner(threading.Thread):
648
648
  num_slots = 1
649
649
  num_extractors = 1
650
650
  num_segmenters = 1
651
- elif seg_cls.hardware_processor == "cpu": # CPU segmenter
651
+ elif seg_cls.hardware_processor == "cpu": # MPO segmenter
652
652
  # We could in principle set the number of slots to one and
653
653
  # have both number of extractors and number of segmenters set
654
654
  # to the total number of CPUs. However, we would need more RAM
@@ -10,7 +10,7 @@ import warnings
10
10
 
11
11
  #: Increment this string if there are breaking changes that make
12
12
  #: previous pipelines unreproducible.
13
- DCNUM_PPID_GENERATION = "9"
13
+ DCNUM_PPID_GENERATION = "10"
14
14
 
15
15
 
16
16
  class ClassWithPPIDCapabilities(Protocol):
@@ -1,6 +1,6 @@
1
1
  # flake8: noqa: F401
2
2
  from .segmenter import Segmenter, get_available_segmenters
3
- from .segmenter_cpu import CPUSegmenter
4
- from .segmenter_gpu import GPUSegmenter
3
+ from .segmenter_mpo import MPOSegmenter
4
+ from .segmenter_sto import STOSegmenter
5
5
  from .segmenter_manager_thread import SegmenterManagerThread
6
6
  from . import segm_thresh
@@ -1,7 +1,7 @@
1
- from .segmenter_cpu import CPUSegmenter
1
+ from .segmenter_mpo import MPOSegmenter
2
2
 
3
3
 
4
- class SegmentThresh(CPUSegmenter):
4
+ class SegmentThresh(MPOSegmenter):
5
5
  mask_postprocessing = True
6
6
  mask_default_kwargs = {
7
7
  "clear_border": True,
@@ -11,8 +11,8 @@ class SegmentThresh(CPUSegmenter):
11
11
  requires_background_correction = True
12
12
 
13
13
  @staticmethod
14
- def segment_approach(image, *,
15
- thresh: float = -6):
14
+ def segment_algorithm(image, *,
15
+ thresh: float = -6):
16
16
  """Mask retrieval using basic thresholding
17
17
 
18
18
  Parameters
@@ -18,7 +18,7 @@ class Segmenter(abc.ABC):
18
18
  hardware_processor = "none"
19
19
  #: Whether to enable mask post-processing. If disabled, you should
20
20
  #: make sure that your mask is properly defined and cleaned or you
21
- #: have to call `process_mask` in your `segment_approach` implementation.
21
+ #: have to call `process_mask` in your `segment_algorithm` implementation.
22
22
  mask_postprocessing = True
23
23
  #: Default keyword arguments for mask post-processing. See `process_mask`
24
24
  #: for available options.
@@ -31,7 +31,13 @@ class Segmenter(abc.ABC):
31
31
  kwargs_mask: Dict = None,
32
32
  debug: bool = False,
33
33
  **kwargs):
34
- """Base segmenter
34
+ """Base segmenter class
35
+
36
+ This is the base segmenter class for the multiprocessing operation
37
+ segmenter :class:`.MPOSegmenter` (multiple subprocesses are spawned
38
+ and each of them works on a queue of images) and the single-threaded
39
+ operation segmenter :class:`.STOSegmenter` (e.g. for batch
40
+ segmentation on a GPU).
35
41
 
36
42
  Parameters
37
43
  ----------
@@ -45,7 +51,7 @@ class Segmenter(abc.ABC):
45
51
  self.debug = debug
46
52
  self.logger = logging.getLogger(__name__).getChild(
47
53
  self.__class__.__name__)
48
- spec = inspect.getfullargspec(self.segment_approach)
54
+ spec = inspect.getfullargspec(self.segment_algorithm)
49
55
  #: custom keyword arguments for the subclassing segmenter
50
56
  self.kwargs = spec.kwonlydefaults or {}
51
57
  self.kwargs.update(kwargs)
@@ -90,7 +96,7 @@ class Segmenter(abc.ABC):
90
96
  KEY:KW_APPROACH:KW_MASK
91
97
 
92
98
  Where KEY is e.g. "legacy" or "watershed", and KW_APPROACH is a
93
- list of keyword arguments for `segment_approach`, e.g.::
99
+ list of keyword arguments for `segment_algorithm`, e.g.::
94
100
 
95
101
  thresh=-6^blur=0
96
102
 
@@ -137,7 +143,7 @@ class Segmenter(abc.ABC):
137
143
 
138
144
  ppid_parts = [
139
145
  cls.get_ppid_code(),
140
- kwargs_to_ppid(cls, "segment_approach", kwargs),
146
+ kwargs_to_ppid(cls, "segment_algorithm", kwargs),
141
147
  ]
142
148
 
143
149
  if cls.mask_postprocessing:
@@ -161,7 +167,7 @@ class Segmenter(abc.ABC):
161
167
  raise ValueError(
162
168
  f"Could not find segmenter '{code}'!")
163
169
  kwargs = ppid_to_kwargs(cls=cls,
164
- method="segment_approach",
170
+ method="segment_algorithm",
165
171
  ppid=pp_kwargs)
166
172
  if cls.mask_postprocessing:
167
173
  pp_kwargs_mask = ppid_parts[2]
@@ -196,7 +202,7 @@ class Segmenter(abc.ABC):
196
202
  if > 0, perform a binary closing with a disk
197
203
  of that radius in pixels
198
204
  """
199
- if labels.dtype == np.bool_:
205
+ if labels.dtype == bool:
200
206
  # Convert mask image to labels
201
207
  labels, _ = ndi.label(
202
208
  input=labels,
@@ -268,53 +274,71 @@ class Segmenter(abc.ABC):
268
274
 
269
275
  return labels
270
276
 
271
- def segment_chunk(self, image_data, chunk, bg_off=None):
272
- """Return the integer labels for one `image_data` chunk"""
273
- data = image_data.get_chunk(chunk)
274
- if bg_off is not None:
275
- bg_off_chunk = bg_off[image_data.get_chunk_slice(chunk)]
276
- data = data - bg_off_chunk.reshape(-1, 1, 1)
277
- return self.segment_batch(data)
278
-
279
- def segment_frame(self, image):
280
- """Return the integer label image for `index`"""
281
- segm_wrap = self.segment_frame_wrapper()
282
- # obtain mask or label
283
- mol = segm_wrap(image)
284
- if mol.dtype == bool:
285
- # convert mask to label
286
- labels, _ = ndi.label(
287
- input=mol,
288
- structure=ndi.generate_binary_structure(2, 2))
289
- else:
290
- labels = mol
291
- # optional postprocessing
292
- if self.mask_postprocessing:
293
- labels = self.process_mask(labels, **self.kwargs_mask)
294
- return labels
277
+ @staticmethod
278
+ @abc.abstractmethod
279
+ def segment_algorithm(image):
280
+ """The segmentation algorithm implemented in the subclass
281
+
282
+ Perform segmentation and return integer label or binary mask image
283
+ """
295
284
 
296
285
  @functools.cache
297
- def segment_frame_wrapper(self):
286
+ def segment_algorithm_wrapper(self):
287
+ """Wraps `self.segment_algorithm` to only accept an image
288
+
289
+ The static method `self.segment_algorithm` may optionally accept
290
+ keyword arguments `self.kwargs`. This wrapper returns the
291
+ wrapped method that only accepts the image as an argument. This
292
+ makes sense if you want to unify
293
+ """
298
294
  if self.kwargs:
299
295
  # For segmenters that accept keyword arguments.
300
- segm_wrap = functools.partial(self.segment_approach,
296
+ segm_wrap = functools.partial(self.segment_algorithm,
301
297
  **self.kwargs)
302
298
  else:
303
299
  # For segmenters that don't accept keyword arguments.
304
- segm_wrap = self.segment_approach
300
+ segm_wrap = self.segment_algorithm
305
301
  return segm_wrap
306
302
 
307
- @staticmethod
308
303
  @abc.abstractmethod
309
- def segment_approach(image):
310
- """Perform segmentation and return integer label or binary mask image
304
+ def segment_batch(self, images, start=None, stop=None, bg_off=None):
305
+ """Return the integer labels for an entire batch
306
+
307
+ This is implemented in the MPO and STO segmenters.
308
+ """
309
+
310
+ def segment_chunk(self, image_data, chunk, bg_off=None):
311
+ """Return the integer labels for one `image_data` chunk
312
+
313
+ This is a wrapper for `segment_batch`.
311
314
 
312
- This is the approach the subclasses implement.
315
+ Parameters
316
+ ----------
317
+ image_data:
318
+ Instance of dcnum's :class:`.BaseImageChunkCache` with
319
+ the methods `get_chunk` and `get_chunk_slice`.
320
+ chunk: int
321
+ Integer identifying the chunk in `image_data` to segment
322
+ bg_off: ndarray
323
+ Optional 1D array with same length as `image_data` that holds
324
+ additional background offset values that should be subtracted
325
+ from the image data before segmentation. Should only be
326
+ used in combination with segmenters that have
327
+ `requires_background_correction` set to True.
313
328
  """
329
+ images = image_data.get_chunk(chunk)
330
+ if bg_off is not None:
331
+ bg_off_chunk = bg_off[image_data.get_chunk_slice(chunk)]
332
+ else:
333
+ bg_off_chunk = None
334
+ return self.segment_batch(images, bg_off=bg_off_chunk)
314
335
 
315
336
  @abc.abstractmethod
316
- def segment_batch(self, data, start=None, stop=None):
317
- """Return the integer labels for an entire batch"""
337
+ def segment_single(self, image):
338
+ """Return the integer label for one image
339
+
340
+ This is implemented in the MPO and STO segmenters.
341
+ """
318
342
 
319
343
 
320
344
  @functools.cache
@@ -8,7 +8,7 @@ import numpy as np
8
8
  from ..read.cache import HDF5ImageCache, ImageCorrCache
9
9
 
10
10
  from .segmenter import Segmenter
11
- from .segmenter_cpu import CPUSegmenter
11
+ from .segmenter_mpo import MPOSegmenter
12
12
 
13
13
 
14
14
  class SegmenterManagerThread(threading.Thread):
@@ -66,7 +66,8 @@ class SegmenterManagerThread(threading.Thread):
66
66
  #: Image data which is being segmented
67
67
  self.image_data = image_data
68
68
  #: Additional, optional background offset
69
- self.bg_off = bg_off
69
+ self.bg_off = (
70
+ bg_off if self.segmenter.requires_background_correction else None)
70
71
  #: Slot states
71
72
  self.slot_states = slot_states
72
73
  #: Current slot chunk index for the slot states
@@ -128,7 +129,7 @@ class SegmenterManagerThread(threading.Thread):
128
129
  self.t_count += time.monotonic() - t1
129
130
 
130
131
  # Cleanup
131
- if isinstance(self.segmenter, CPUSegmenter):
132
+ if isinstance(self.segmenter, MPOSegmenter):
132
133
  # Join the segmentation workers.
133
134
  self.segmenter.join_workers()
134
135
 
@@ -5,6 +5,7 @@ import threading
5
5
  from typing import Dict
6
6
 
7
7
  import numpy as np
8
+ import scipy.ndimage as ndi
8
9
 
9
10
  from .segmenter import Segmenter
10
11
 
@@ -14,7 +15,7 @@ from .segmenter import Segmenter
14
15
  mp_spawn = mp.get_context('spawn')
15
16
 
16
17
 
17
- class CPUSegmenter(Segmenter, abc.ABC):
18
+ class MPOSegmenter(Segmenter, abc.ABC):
18
19
  hardware_processor = "cpu"
19
20
 
20
21
  def __init__(self,
@@ -23,7 +24,7 @@ class CPUSegmenter(Segmenter, abc.ABC):
23
24
  kwargs_mask: Dict = None,
24
25
  debug: bool = False,
25
26
  **kwargs):
26
- """CPU base segmenter
27
+ """Segmenter with multiprocessing operation
27
28
 
28
29
  Parameters
29
30
  ----------
@@ -32,17 +33,23 @@ class CPUSegmenter(Segmenter, abc.ABC):
32
33
  debug: bool
33
34
  Debugging parameters
34
35
  kwargs:
35
- Additional, optional keyword arguments for `segment_approach`
36
+ Additional, optional keyword arguments for `segment_algorithm`
36
37
  defined in the subclass.
37
38
  """
38
- super(CPUSegmenter, self).__init__(kwargs_mask=kwargs_mask,
39
+ super(MPOSegmenter, self).__init__(kwargs_mask=kwargs_mask,
39
40
  debug=debug,
40
41
  **kwargs)
41
42
  self.num_workers = num_workers or mp.cpu_count()
43
+ # batch input image data
42
44
  self.mp_image_raw = None
43
45
  self._mp_image_np = None
46
+ # batch output image data
44
47
  self.mp_labels_raw = None
45
48
  self._mp_labels_np = None
49
+ # batch image background offset
50
+ self.mp_bg_off_raw = None
51
+ self._mp_bg_off_np = None
52
+ # workers
46
53
  self._mp_workers = []
47
54
  # Image shape of the input array
48
55
  self.image_shape = None
@@ -78,6 +85,7 @@ class CPUSegmenter(Segmenter, abc.ABC):
78
85
  del state["logger"]
79
86
  del state["_mp_image_np"]
80
87
  del state["_mp_labels_np"]
88
+ del state["_mp_bg_off_np"]
81
89
  del state["_mp_workers"]
82
90
  return state
83
91
 
@@ -86,26 +94,26 @@ class CPUSegmenter(Segmenter, abc.ABC):
86
94
  self.__dict__.update(state)
87
95
 
88
96
  @staticmethod
89
- def _create_shared_array(image_shape, batch_size, dtype):
97
+ def _create_shared_array(array_shape, batch_size, dtype):
90
98
  """Return raw and numpy-view on shared array
91
99
 
92
100
  Parameters
93
101
  ----------
94
- image_shape: tuple of int
102
+ array_shape: tuple of int
95
103
  Shape of one single image in the array
96
104
  batch_size: int
97
105
  Number of images in the array
98
106
  dtype:
99
107
  numpy dtype
100
108
  """
101
- sx, sy = image_shape
102
109
  ctype = np.ctypeslib.as_ctypes_type(dtype)
103
- sa_raw = mp_spawn.RawArray(ctype, int(sx * sy * batch_size))
110
+ sa_raw = mp_spawn.RawArray(ctype,
111
+ int(np.prod(array_shape) * batch_size))
104
112
  # Convert the RawArray to something we can write to fast
105
113
  # (similar to memory view, but without having to cast) using
106
114
  # np.ctypeslib.as_array. See discussion in
107
115
  # https://stackoverflow.com/questions/37705974
108
- sa_np = np.ctypeslib.as_array(sa_raw).reshape(batch_size, sx, sy)
116
+ sa_np = np.ctypeslib.as_array(sa_raw).reshape(batch_size, *array_shape)
109
117
  return sa_raw, sa_np
110
118
 
111
119
  @property
@@ -127,39 +135,49 @@ class CPUSegmenter(Segmenter, abc.ABC):
127
135
  [w.join() for w in self._mp_workers]
128
136
 
129
137
  def segment_batch(self,
130
- image_data: np.ndarray,
138
+ images: np.ndarray,
131
139
  start: int = None,
132
- stop: int = None):
133
- """Perform batch segmentation of `image_data`
140
+ stop: int = None,
141
+ bg_off: np.ndarray = None,
142
+ ):
143
+ """Perform batch segmentation of `images`
144
+
145
+ Before segmentation, an optional background offset correction with
146
+ `bg_off` is performed. After segmentation, mask postprocessing is
147
+ performed according to the class definition.
134
148
 
135
149
  Parameters
136
150
  ----------
137
- image_data: 3d np.ndarray
151
+ images: 3d np.ndarray
138
152
  The time-series image data. First axis is time.
139
153
  start: int
140
- First index to analyze in `image_data`
154
+ First index to analyze in `images`
141
155
  stop: int
142
- Index after the last index to analyze in `image_data`
156
+ Index after the last index to analyze in `images`
157
+ bg_off: 1D np.ndarray
158
+ Optional 1D numpy array with background offset
143
159
 
144
160
  Notes
145
161
  -----
146
162
  - If the segmentation algorithm only accepts background-corrected
147
- images, then `image_data` must already be background-corrected.
163
+ images, then `images` must already be background-corrected,
164
+ except for the optional `bg_off`.
148
165
  """
149
166
  if stop is None or start is None:
150
167
  start = 0
151
- stop = len(image_data)
168
+ stop = len(images)
152
169
 
153
170
  batch_size = stop - start
154
- size = np.prod(image_data.shape[1:]) * batch_size
171
+ size = np.prod(images.shape[1:]) * batch_size
155
172
 
156
173
  if self.image_shape is None:
157
- self.image_shape = image_data[0].shape
174
+ self.image_shape = images[0].shape
158
175
 
159
176
  if self._mp_image_np is not None and self._mp_image_np.size != size:
160
177
  # reset image data
161
178
  self._mp_image_np = None
162
179
  self._mp_labels_np = None
180
+ self._mp_bg_off_np = None
163
181
  # TODO: If only the batch_size changes, don't
164
182
  # reinitialize the workers. Otherwise, the final rest of
165
183
  # analyzing a dataset would always take a little longer.
@@ -168,31 +186,47 @@ class CPUSegmenter(Segmenter, abc.ABC):
168
186
  self.mp_batch_index.value = -1
169
187
  self.mp_shutdown.value = 0
170
188
 
189
+ if bg_off is not None:
190
+ if not self.requires_background_correction:
191
+ raise ValueError(f"The segmenter {self.__class__.__name__} "
192
+ f"does not employ background correction, "
193
+ f"but the `bg_off` keyword argument was "
194
+ f"passed to `segment_chunk`. Please check "
195
+ f"your analysis pipeline.")
196
+ # background offset
197
+ if self._mp_bg_off_np is None:
198
+ self.mp_bg_off_raw, self._mp_bg_off_np = \
199
+ self._create_shared_array(
200
+ array_shape=(stop - start,),
201
+ batch_size=batch_size,
202
+ dtype=np.float64)
203
+ self._mp_bg_off_np[:] = bg_off[start:stop]
204
+
205
+ # input images
171
206
  if self._mp_image_np is None:
172
207
  self.mp_image_raw, self._mp_image_np = self._create_shared_array(
173
- image_shape=self.image_shape,
208
+ array_shape=self.image_shape,
174
209
  batch_size=batch_size,
175
- dtype=image_data.dtype,
210
+ dtype=images.dtype,
176
211
  )
212
+ self._mp_image_np[:] = images[start:stop]
177
213
 
214
+ # output labels
178
215
  if self._mp_labels_np is None:
179
216
  self.mp_labels_raw, self._mp_labels_np = self._create_shared_array(
180
- image_shape=self.image_shape,
217
+ array_shape=self.image_shape,
181
218
  batch_size=batch_size,
182
219
  dtype=np.uint16,
183
220
  )
184
221
 
185
- # populate image data
186
- self._mp_image_np[:] = image_data[start:stop]
187
-
188
222
  # Create the workers
189
223
  if self.debug:
190
- worker_cls = CPUSegmenterWorkerThread
224
+ worker_cls = MPOSegmenterWorkerThread
191
225
  num_workers = 1
192
226
  self.logger.debug("Running with one worker in main thread")
193
227
  else:
194
- worker_cls = CPUSegmenterWorkerProcess
195
- num_workers = min(self.num_workers, image_data.shape[0])
228
+ worker_cls = MPOSegmenterWorkerProcess
229
+ num_workers = min(self.num_workers, images.shape[0])
196
230
  self.logger.debug(f"Running with {num_workers} workers")
197
231
 
198
232
  if not self._mp_workers:
@@ -224,8 +258,33 @@ class CPUSegmenter(Segmenter, abc.ABC):
224
258
 
225
259
  return self._mp_labels_np
226
260
 
261
+ def segment_single(self, image, bg_off: float = None):
262
+ """Return the integer label image for an input image
263
+
264
+ Before segmentation, an optional background offset correction with
265
+ `bg_off` is performed. After segmentation, mask postprocessing is
266
+ performed according to the class definition.
267
+ """
268
+ segm_wrap = self.segment_algorithm_wrapper()
269
+ # optional subtraction of background offset
270
+ if bg_off is not None:
271
+ image = image - bg_off
272
+ # obtain mask or label
273
+ mol = segm_wrap(image)
274
+ if mol.dtype == bool:
275
+ # convert mask to labels
276
+ labels, _ = ndi.label(
277
+ input=mol,
278
+ structure=ndi.generate_binary_structure(2, 2))
279
+ else:
280
+ labels = mol
281
+ # optional mask/label postprocessing
282
+ if self.mask_postprocessing:
283
+ labels = self.process_mask(labels, **self.kwargs_mask)
284
+ return labels
285
+
227
286
 
228
- class CPUSegmenterWorker:
287
+ class MPOSegmenterWorker:
229
288
  def __init__(self,
230
289
  segmenter,
231
290
  sl_start: int,
@@ -235,7 +294,7 @@ class CPUSegmenterWorker:
235
294
 
236
295
  Parameters
237
296
  ----------
238
- segmenter: CPUSegmenter
297
+ segmenter: MPOSegmenter
239
298
  The segmentation instance
240
299
  sl_start: int
241
300
  Start of slice of input array to process
@@ -243,7 +302,7 @@ class CPUSegmenterWorker:
243
302
  Stop of slice of input array to process
244
303
  """
245
304
  # Must call super init, otherwise Thread or Process are not initialized
246
- super(CPUSegmenterWorker, self).__init__()
305
+ super(MPOSegmenterWorker, self).__init__()
247
306
  self.segmenter = segmenter
248
307
  # Value incrementing the batch index. Starts with 0 and is
249
308
  # incremented every time :func:`Segmenter.segment_batch` is
@@ -255,8 +314,10 @@ class CPUSegmenterWorker:
255
314
  # Shutdown bit tells workers to stop when set to != 0
256
315
  self.shutdown = segmenter.mp_shutdown
257
316
  # The image data for segmentation
258
- self.image_data_raw = segmenter.mp_image_raw
259
- # Boolean mask array
317
+ self.image_arr_raw = segmenter.mp_image_raw
318
+ # Background data offset
319
+ self.bg_off = segmenter.mp_bg_off_raw
320
+ # Integer output label array
260
321
  self.labels_data_raw = segmenter.mp_labels_raw
261
322
  # The shape of one image
262
323
  self.image_shape = segmenter.image_shape
@@ -268,10 +329,14 @@ class CPUSegmenterWorker:
268
329
  # We have to create the numpy-versions of the mp.RawArrays here,
269
330
  # otherwise we only get some kind of copy in the new process
270
331
  # when we use "spawn" instead of "fork".
271
- labels_data = np.ctypeslib.as_array(self.labels_data_raw).reshape(
332
+ labels_arr = np.ctypeslib.as_array(self.labels_data_raw).reshape(
272
333
  -1, self.image_shape[0], self.image_shape[1])
273
- image_data = np.ctypeslib.as_array(self.image_data_raw).reshape(
334
+ image_arr = np.ctypeslib.as_array(self.image_arr_raw).reshape(
274
335
  -1, self.image_shape[0], self.image_shape[1])
336
+ if self.bg_off is not None:
337
+ bg_off_data = np.ctypeslib.as_array(self.bg_off)
338
+ else:
339
+ bg_off_data = None
275
340
 
276
341
  idx = self.sl_start
277
342
  itr = 0 # current iteration (incremented when we reach self.sl_stop)
@@ -285,8 +350,13 @@ class CPUSegmenterWorker:
285
350
  with self.batch_worker:
286
351
  self.batch_worker.value += 1
287
352
  else:
288
- labels_data[idx, :, :] = self.segmenter.segment_frame(
289
- image_data[idx])
353
+ if bg_off_data is None:
354
+ bg_off = None
355
+ else:
356
+ bg_off = bg_off_data[idx]
357
+
358
+ labels_arr[idx, :, :] = self.segmenter.segment_single(
359
+ image=image_arr[idx], bg_off=bg_off)
290
360
  idx += 1
291
361
  elif self.shutdown.value:
292
362
  break
@@ -295,11 +365,11 @@ class CPUSegmenterWorker:
295
365
  time.sleep(.01)
296
366
 
297
367
 
298
- class CPUSegmenterWorkerProcess(CPUSegmenterWorker, mp_spawn.Process):
368
+ class MPOSegmenterWorkerProcess(MPOSegmenterWorker, mp_spawn.Process):
299
369
  def __init__(self, *args, **kwargs):
300
- super(CPUSegmenterWorkerProcess, self).__init__(*args, **kwargs)
370
+ super(MPOSegmenterWorkerProcess, self).__init__(*args, **kwargs)
301
371
 
302
372
 
303
- class CPUSegmenterWorkerThread(CPUSegmenterWorker, threading.Thread):
373
+ class MPOSegmenterWorkerThread(MPOSegmenterWorker, threading.Thread):
304
374
  def __init__(self, *args, **kwargs):
305
- super(CPUSegmenterWorkerThread, self).__init__(*args, **kwargs)
375
+ super(MPOSegmenterWorkerThread, self).__init__(*args, **kwargs)