dcnum 0.21.3__tar.gz → 0.22.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dcnum might be problematic. Click here for more details.
- {dcnum-0.21.3 → dcnum-0.22.0}/CHANGELOG +10 -1
- {dcnum-0.21.3/src/dcnum.egg-info → dcnum-0.22.0}/PKG-INFO +1 -1
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/_version.py +2 -2
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/event_extractor_manager_thread.py +1 -1
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/logic/ctrl.py +1 -1
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/meta/ppid.py +1 -1
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/segm/__init__.py +2 -2
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/segm/segm_thresh.py +4 -4
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/segm/segmenter.py +64 -40
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/segm/segmenter_manager_thread.py +4 -3
- dcnum-0.21.3/src/dcnum/segm/segmenter_cpu.py → dcnum-0.22.0/src/dcnum/segm/segmenter_mpo.py +111 -41
- dcnum-0.22.0/src/dcnum/segm/segmenter_sto.py +110 -0
- {dcnum-0.21.3 → dcnum-0.22.0/src/dcnum.egg-info}/PKG-INFO +1 -1
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum.egg-info/SOURCES.txt +4 -2
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/helper_methods.py +29 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_ppid_segm.py +1 -1
- dcnum-0.22.0/tests/test_segm_base.py +143 -0
- dcnum-0.21.3/tests/test_segm_base.py → dcnum-0.22.0/tests/test_segm_mpo.py +132 -167
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_segm_no_mask_proc.py +2 -2
- dcnum-0.22.0/tests/test_segm_sto.py +294 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_segm_thresh.py +4 -4
- dcnum-0.21.3/src/dcnum/segm/segmenter_gpu.py +0 -63
- {dcnum-0.21.3 → dcnum-0.22.0}/.github/workflows/check.yml +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/.github/workflows/deploy_pypi.yml +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/.gitignore +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/.readthedocs.yml +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/LICENSE +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/README.rst +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/docs/conf.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/docs/extensions/github_changelog.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/docs/index.rst +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/docs/requirements.txt +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/pyproject.toml +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/setup.cfg +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/__init__.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/__init__.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_background/__init__.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_background/base.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_background/bg_copy.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_background/bg_roll_median.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_background/bg_sparse_median.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_brightness/__init__.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_brightness/bright_all.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_brightness/common.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_contour/__init__.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_contour/contour.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_contour/moments.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_contour/volume.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_texture/__init__.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_texture/common.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/feat_texture/tex_all.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/gate.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/feat/queue_event_extractor.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/logic/__init__.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/logic/job.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/logic/json_encoder.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/meta/__init__.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/meta/paths.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/read/__init__.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/read/cache.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/read/const.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/read/hdf5_data.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/read/mapped.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/write/__init__.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/write/deque_writer_thread.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/write/queue_collector_thread.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum/write/writer.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum.egg-info/dependency_links.txt +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum.egg-info/requires.txt +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/src/dcnum.egg-info/top_level.txt +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/conftest.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/data/fmt-hdf5_cytoshot_extended-moments-features.zip +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/data/fmt-hdf5_cytoshot_full-features_2023.zip +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/data/fmt-hdf5_cytoshot_full-features_2024.zip +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/data/fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/data/fmt-hdf5_shapein_empty.zip +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/data/fmt-hdf5_shapein_raw-with-variable-length-logs.zip +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/requirements.txt +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_background_base.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_background_bg_copy.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_background_bg_roll_median.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_background_bg_sparsemed.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_brightness.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_event_extractor_manager.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_gate.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_haralick.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_moments_based.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_moments_based_extended.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_feat_volume.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_init.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_logic_job.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_logic_join.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_logic_json.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_logic_pipeline.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_paths.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_ppid_base.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_ppid_bg.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_ppid_data.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_ppid_feat.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_meta_ppid_gate.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_read_basin.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_read_concat_hdf5.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_read_hdf5.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_read_hdf5_basins.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_read_hdf5_index_mapping.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_write_deque_writer_thread.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_write_queue_collector_thread.py +0 -0
- {dcnum-0.21.3 → dcnum-0.22.0}/tests/test_write_writer.py +0 -0
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
0.22.0
|
|
2
|
+
- fix: GPUSegmenter did not perform mask postprocessing
|
|
3
|
+
(this was not actually fixed in 0.21.0)
|
|
4
|
+
- tests: implement mock STO segmenter for testing (#3)
|
|
5
|
+
- tests: add tests for bg_off in segmentation
|
|
6
|
+
- ref: increment DCNUM_PPID_GENERATION to 10
|
|
7
|
+
- ref: renamed CPUSegmenter to MPOSegmenter
|
|
8
|
+
- ref: renamed GPUSegmenter to STOSegmenter
|
|
9
|
+
0.21.4
|
|
10
|
+
- fix: division by zero error when computing stall time
|
|
1
11
|
0.21.3
|
|
2
12
|
- fix: negative stall time in log messages
|
|
3
13
|
0.21.2
|
|
@@ -5,7 +15,6 @@
|
|
|
5
15
|
0.21.1
|
|
6
16
|
- enh: support boolean images in mask postprocessing
|
|
7
17
|
0.21.0
|
|
8
|
-
- fix: GPUSegmenter did not perform mask postprocessing
|
|
9
18
|
- ref: enable mask postprocessing by default for all segmenters
|
|
10
19
|
- ref: increment DCNUM_PPID_GENERATION to 9
|
|
11
20
|
0.20.4
|
|
@@ -98,7 +98,7 @@ class EventExtractorManagerThread(threading.Thread):
|
|
|
98
98
|
if (ldq := len(self.writer_dq)) > 1000:
|
|
99
99
|
time.sleep(1)
|
|
100
100
|
ldq2 = len(self.writer_dq)
|
|
101
|
-
stall_time = max(0., (ldq2 - 200) / (ldq - ldq2))
|
|
101
|
+
stall_time = max(0., (ldq2 - 200) / ((ldq - ldq2) or 1))
|
|
102
102
|
time.sleep(stall_time)
|
|
103
103
|
self.logger.warning(
|
|
104
104
|
f"Stalled {stall_time + 1:.1f}s for slow writer "
|
|
@@ -648,7 +648,7 @@ class DCNumJobRunner(threading.Thread):
|
|
|
648
648
|
num_slots = 1
|
|
649
649
|
num_extractors = 1
|
|
650
650
|
num_segmenters = 1
|
|
651
|
-
elif seg_cls.hardware_processor == "cpu": #
|
|
651
|
+
elif seg_cls.hardware_processor == "cpu": # MPO segmenter
|
|
652
652
|
# We could in principle set the number of slots to one and
|
|
653
653
|
# have both number of extractors and number of segmenters set
|
|
654
654
|
# to the total number of CPUs. However, we would need more RAM
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# flake8: noqa: F401
|
|
2
2
|
from .segmenter import Segmenter, get_available_segmenters
|
|
3
|
-
from .
|
|
4
|
-
from .
|
|
3
|
+
from .segmenter_mpo import MPOSegmenter
|
|
4
|
+
from .segmenter_sto import STOSegmenter
|
|
5
5
|
from .segmenter_manager_thread import SegmenterManagerThread
|
|
6
6
|
from . import segm_thresh
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
from .
|
|
1
|
+
from .segmenter_mpo import MPOSegmenter
|
|
2
2
|
|
|
3
3
|
|
|
4
|
-
class SegmentThresh(
|
|
4
|
+
class SegmentThresh(MPOSegmenter):
|
|
5
5
|
mask_postprocessing = True
|
|
6
6
|
mask_default_kwargs = {
|
|
7
7
|
"clear_border": True,
|
|
@@ -11,8 +11,8 @@ class SegmentThresh(CPUSegmenter):
|
|
|
11
11
|
requires_background_correction = True
|
|
12
12
|
|
|
13
13
|
@staticmethod
|
|
14
|
-
def
|
|
15
|
-
|
|
14
|
+
def segment_algorithm(image, *,
|
|
15
|
+
thresh: float = -6):
|
|
16
16
|
"""Mask retrieval using basic thresholding
|
|
17
17
|
|
|
18
18
|
Parameters
|
|
@@ -18,7 +18,7 @@ class Segmenter(abc.ABC):
|
|
|
18
18
|
hardware_processor = "none"
|
|
19
19
|
#: Whether to enable mask post-processing. If disabled, you should
|
|
20
20
|
#: make sure that your mask is properly defined and cleaned or you
|
|
21
|
-
#: have to call `process_mask` in your `
|
|
21
|
+
#: have to call `process_mask` in your `segment_algorithm` implementation.
|
|
22
22
|
mask_postprocessing = True
|
|
23
23
|
#: Default keyword arguments for mask post-processing. See `process_mask`
|
|
24
24
|
#: for available options.
|
|
@@ -31,7 +31,13 @@ class Segmenter(abc.ABC):
|
|
|
31
31
|
kwargs_mask: Dict = None,
|
|
32
32
|
debug: bool = False,
|
|
33
33
|
**kwargs):
|
|
34
|
-
"""Base segmenter
|
|
34
|
+
"""Base segmenter class
|
|
35
|
+
|
|
36
|
+
This is the base segmenter class for the multiprocessing operation
|
|
37
|
+
segmenter :class:`.MPOSegmenter` (multiple subprocesses are spawned
|
|
38
|
+
and each of them works on a queue of images) and the single-threaded
|
|
39
|
+
operation segmenter :class:`.STOSegmenter` (e.g. for batch
|
|
40
|
+
segmentation on a GPU).
|
|
35
41
|
|
|
36
42
|
Parameters
|
|
37
43
|
----------
|
|
@@ -45,7 +51,7 @@ class Segmenter(abc.ABC):
|
|
|
45
51
|
self.debug = debug
|
|
46
52
|
self.logger = logging.getLogger(__name__).getChild(
|
|
47
53
|
self.__class__.__name__)
|
|
48
|
-
spec = inspect.getfullargspec(self.
|
|
54
|
+
spec = inspect.getfullargspec(self.segment_algorithm)
|
|
49
55
|
#: custom keyword arguments for the subclassing segmenter
|
|
50
56
|
self.kwargs = spec.kwonlydefaults or {}
|
|
51
57
|
self.kwargs.update(kwargs)
|
|
@@ -90,7 +96,7 @@ class Segmenter(abc.ABC):
|
|
|
90
96
|
KEY:KW_APPROACH:KW_MASK
|
|
91
97
|
|
|
92
98
|
Where KEY is e.g. "legacy" or "watershed", and KW_APPROACH is a
|
|
93
|
-
list of keyword arguments for `
|
|
99
|
+
list of keyword arguments for `segment_algorithm`, e.g.::
|
|
94
100
|
|
|
95
101
|
thresh=-6^blur=0
|
|
96
102
|
|
|
@@ -137,7 +143,7 @@ class Segmenter(abc.ABC):
|
|
|
137
143
|
|
|
138
144
|
ppid_parts = [
|
|
139
145
|
cls.get_ppid_code(),
|
|
140
|
-
kwargs_to_ppid(cls, "
|
|
146
|
+
kwargs_to_ppid(cls, "segment_algorithm", kwargs),
|
|
141
147
|
]
|
|
142
148
|
|
|
143
149
|
if cls.mask_postprocessing:
|
|
@@ -161,7 +167,7 @@ class Segmenter(abc.ABC):
|
|
|
161
167
|
raise ValueError(
|
|
162
168
|
f"Could not find segmenter '{code}'!")
|
|
163
169
|
kwargs = ppid_to_kwargs(cls=cls,
|
|
164
|
-
method="
|
|
170
|
+
method="segment_algorithm",
|
|
165
171
|
ppid=pp_kwargs)
|
|
166
172
|
if cls.mask_postprocessing:
|
|
167
173
|
pp_kwargs_mask = ppid_parts[2]
|
|
@@ -196,7 +202,7 @@ class Segmenter(abc.ABC):
|
|
|
196
202
|
if > 0, perform a binary closing with a disk
|
|
197
203
|
of that radius in pixels
|
|
198
204
|
"""
|
|
199
|
-
if labels.dtype ==
|
|
205
|
+
if labels.dtype == bool:
|
|
200
206
|
# Convert mask image to labels
|
|
201
207
|
labels, _ = ndi.label(
|
|
202
208
|
input=labels,
|
|
@@ -268,53 +274,71 @@ class Segmenter(abc.ABC):
|
|
|
268
274
|
|
|
269
275
|
return labels
|
|
270
276
|
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
def segment_frame(self, image):
|
|
280
|
-
"""Return the integer label image for `index`"""
|
|
281
|
-
segm_wrap = self.segment_frame_wrapper()
|
|
282
|
-
# obtain mask or label
|
|
283
|
-
mol = segm_wrap(image)
|
|
284
|
-
if mol.dtype == bool:
|
|
285
|
-
# convert mask to label
|
|
286
|
-
labels, _ = ndi.label(
|
|
287
|
-
input=mol,
|
|
288
|
-
structure=ndi.generate_binary_structure(2, 2))
|
|
289
|
-
else:
|
|
290
|
-
labels = mol
|
|
291
|
-
# optional postprocessing
|
|
292
|
-
if self.mask_postprocessing:
|
|
293
|
-
labels = self.process_mask(labels, **self.kwargs_mask)
|
|
294
|
-
return labels
|
|
277
|
+
@staticmethod
|
|
278
|
+
@abc.abstractmethod
|
|
279
|
+
def segment_algorithm(image):
|
|
280
|
+
"""The segmentation algorithm implemented in the subclass
|
|
281
|
+
|
|
282
|
+
Perform segmentation and return integer label or binary mask image
|
|
283
|
+
"""
|
|
295
284
|
|
|
296
285
|
@functools.cache
|
|
297
|
-
def
|
|
286
|
+
def segment_algorithm_wrapper(self):
|
|
287
|
+
"""Wraps `self.segment_algorithm` to only accept an image
|
|
288
|
+
|
|
289
|
+
The static method `self.segment_algorithm` may optionally accept
|
|
290
|
+
keyword arguments `self.kwargs`. This wrapper returns the
|
|
291
|
+
wrapped method that only accepts the image as an argument. This
|
|
292
|
+
makes sense if you want to unify
|
|
293
|
+
"""
|
|
298
294
|
if self.kwargs:
|
|
299
295
|
# For segmenters that accept keyword arguments.
|
|
300
|
-
segm_wrap = functools.partial(self.
|
|
296
|
+
segm_wrap = functools.partial(self.segment_algorithm,
|
|
301
297
|
**self.kwargs)
|
|
302
298
|
else:
|
|
303
299
|
# For segmenters that don't accept keyword arguments.
|
|
304
|
-
segm_wrap = self.
|
|
300
|
+
segm_wrap = self.segment_algorithm
|
|
305
301
|
return segm_wrap
|
|
306
302
|
|
|
307
|
-
@staticmethod
|
|
308
303
|
@abc.abstractmethod
|
|
309
|
-
def
|
|
310
|
-
"""
|
|
304
|
+
def segment_batch(self, images, start=None, stop=None, bg_off=None):
|
|
305
|
+
"""Return the integer labels for an entire batch
|
|
306
|
+
|
|
307
|
+
This is implemented in the MPO and STO segmenters.
|
|
308
|
+
"""
|
|
309
|
+
|
|
310
|
+
def segment_chunk(self, image_data, chunk, bg_off=None):
|
|
311
|
+
"""Return the integer labels for one `image_data` chunk
|
|
312
|
+
|
|
313
|
+
This is a wrapper for `segment_batch`.
|
|
311
314
|
|
|
312
|
-
|
|
315
|
+
Parameters
|
|
316
|
+
----------
|
|
317
|
+
image_data:
|
|
318
|
+
Instance of dcnum's :class:`.BaseImageChunkCache` with
|
|
319
|
+
the methods `get_chunk` and `get_chunk_slice`.
|
|
320
|
+
chunk: int
|
|
321
|
+
Integer identifying the chunk in `image_data` to segment
|
|
322
|
+
bg_off: ndarray
|
|
323
|
+
Optional 1D array with same length as `image_data` that holds
|
|
324
|
+
additional background offset values that should be subtracted
|
|
325
|
+
from the image data before segmentation. Should only be
|
|
326
|
+
used in combination with segmenters that have
|
|
327
|
+
`requires_background_correction` set to True.
|
|
313
328
|
"""
|
|
329
|
+
images = image_data.get_chunk(chunk)
|
|
330
|
+
if bg_off is not None:
|
|
331
|
+
bg_off_chunk = bg_off[image_data.get_chunk_slice(chunk)]
|
|
332
|
+
else:
|
|
333
|
+
bg_off_chunk = None
|
|
334
|
+
return self.segment_batch(images, bg_off=bg_off_chunk)
|
|
314
335
|
|
|
315
336
|
@abc.abstractmethod
|
|
316
|
-
def
|
|
317
|
-
"""Return the integer
|
|
337
|
+
def segment_single(self, image):
|
|
338
|
+
"""Return the integer label for one image
|
|
339
|
+
|
|
340
|
+
This is implemented in the MPO and STO segmenters.
|
|
341
|
+
"""
|
|
318
342
|
|
|
319
343
|
|
|
320
344
|
@functools.cache
|
|
@@ -8,7 +8,7 @@ import numpy as np
|
|
|
8
8
|
from ..read.cache import HDF5ImageCache, ImageCorrCache
|
|
9
9
|
|
|
10
10
|
from .segmenter import Segmenter
|
|
11
|
-
from .
|
|
11
|
+
from .segmenter_mpo import MPOSegmenter
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class SegmenterManagerThread(threading.Thread):
|
|
@@ -66,7 +66,8 @@ class SegmenterManagerThread(threading.Thread):
|
|
|
66
66
|
#: Image data which is being segmented
|
|
67
67
|
self.image_data = image_data
|
|
68
68
|
#: Additional, optional background offset
|
|
69
|
-
self.bg_off =
|
|
69
|
+
self.bg_off = (
|
|
70
|
+
bg_off if self.segmenter.requires_background_correction else None)
|
|
70
71
|
#: Slot states
|
|
71
72
|
self.slot_states = slot_states
|
|
72
73
|
#: Current slot chunk index for the slot states
|
|
@@ -128,7 +129,7 @@ class SegmenterManagerThread(threading.Thread):
|
|
|
128
129
|
self.t_count += time.monotonic() - t1
|
|
129
130
|
|
|
130
131
|
# Cleanup
|
|
131
|
-
if isinstance(self.segmenter,
|
|
132
|
+
if isinstance(self.segmenter, MPOSegmenter):
|
|
132
133
|
# Join the segmentation workers.
|
|
133
134
|
self.segmenter.join_workers()
|
|
134
135
|
|
|
@@ -5,6 +5,7 @@ import threading
|
|
|
5
5
|
from typing import Dict
|
|
6
6
|
|
|
7
7
|
import numpy as np
|
|
8
|
+
import scipy.ndimage as ndi
|
|
8
9
|
|
|
9
10
|
from .segmenter import Segmenter
|
|
10
11
|
|
|
@@ -14,7 +15,7 @@ from .segmenter import Segmenter
|
|
|
14
15
|
mp_spawn = mp.get_context('spawn')
|
|
15
16
|
|
|
16
17
|
|
|
17
|
-
class
|
|
18
|
+
class MPOSegmenter(Segmenter, abc.ABC):
|
|
18
19
|
hardware_processor = "cpu"
|
|
19
20
|
|
|
20
21
|
def __init__(self,
|
|
@@ -23,7 +24,7 @@ class CPUSegmenter(Segmenter, abc.ABC):
|
|
|
23
24
|
kwargs_mask: Dict = None,
|
|
24
25
|
debug: bool = False,
|
|
25
26
|
**kwargs):
|
|
26
|
-
"""
|
|
27
|
+
"""Segmenter with multiprocessing operation
|
|
27
28
|
|
|
28
29
|
Parameters
|
|
29
30
|
----------
|
|
@@ -32,17 +33,23 @@ class CPUSegmenter(Segmenter, abc.ABC):
|
|
|
32
33
|
debug: bool
|
|
33
34
|
Debugging parameters
|
|
34
35
|
kwargs:
|
|
35
|
-
Additional, optional keyword arguments for `
|
|
36
|
+
Additional, optional keyword arguments for `segment_algorithm`
|
|
36
37
|
defined in the subclass.
|
|
37
38
|
"""
|
|
38
|
-
super(
|
|
39
|
+
super(MPOSegmenter, self).__init__(kwargs_mask=kwargs_mask,
|
|
39
40
|
debug=debug,
|
|
40
41
|
**kwargs)
|
|
41
42
|
self.num_workers = num_workers or mp.cpu_count()
|
|
43
|
+
# batch input image data
|
|
42
44
|
self.mp_image_raw = None
|
|
43
45
|
self._mp_image_np = None
|
|
46
|
+
# batch output image data
|
|
44
47
|
self.mp_labels_raw = None
|
|
45
48
|
self._mp_labels_np = None
|
|
49
|
+
# batch image background offset
|
|
50
|
+
self.mp_bg_off_raw = None
|
|
51
|
+
self._mp_bg_off_np = None
|
|
52
|
+
# workers
|
|
46
53
|
self._mp_workers = []
|
|
47
54
|
# Image shape of the input array
|
|
48
55
|
self.image_shape = None
|
|
@@ -78,6 +85,7 @@ class CPUSegmenter(Segmenter, abc.ABC):
|
|
|
78
85
|
del state["logger"]
|
|
79
86
|
del state["_mp_image_np"]
|
|
80
87
|
del state["_mp_labels_np"]
|
|
88
|
+
del state["_mp_bg_off_np"]
|
|
81
89
|
del state["_mp_workers"]
|
|
82
90
|
return state
|
|
83
91
|
|
|
@@ -86,26 +94,26 @@ class CPUSegmenter(Segmenter, abc.ABC):
|
|
|
86
94
|
self.__dict__.update(state)
|
|
87
95
|
|
|
88
96
|
@staticmethod
|
|
89
|
-
def _create_shared_array(
|
|
97
|
+
def _create_shared_array(array_shape, batch_size, dtype):
|
|
90
98
|
"""Return raw and numpy-view on shared array
|
|
91
99
|
|
|
92
100
|
Parameters
|
|
93
101
|
----------
|
|
94
|
-
|
|
102
|
+
array_shape: tuple of int
|
|
95
103
|
Shape of one single image in the array
|
|
96
104
|
batch_size: int
|
|
97
105
|
Number of images in the array
|
|
98
106
|
dtype:
|
|
99
107
|
numpy dtype
|
|
100
108
|
"""
|
|
101
|
-
sx, sy = image_shape
|
|
102
109
|
ctype = np.ctypeslib.as_ctypes_type(dtype)
|
|
103
|
-
sa_raw = mp_spawn.RawArray(ctype,
|
|
110
|
+
sa_raw = mp_spawn.RawArray(ctype,
|
|
111
|
+
int(np.prod(array_shape) * batch_size))
|
|
104
112
|
# Convert the RawArray to something we can write to fast
|
|
105
113
|
# (similar to memory view, but without having to cast) using
|
|
106
114
|
# np.ctypeslib.as_array. See discussion in
|
|
107
115
|
# https://stackoverflow.com/questions/37705974
|
|
108
|
-
sa_np = np.ctypeslib.as_array(sa_raw).reshape(batch_size,
|
|
116
|
+
sa_np = np.ctypeslib.as_array(sa_raw).reshape(batch_size, *array_shape)
|
|
109
117
|
return sa_raw, sa_np
|
|
110
118
|
|
|
111
119
|
@property
|
|
@@ -127,39 +135,49 @@ class CPUSegmenter(Segmenter, abc.ABC):
|
|
|
127
135
|
[w.join() for w in self._mp_workers]
|
|
128
136
|
|
|
129
137
|
def segment_batch(self,
|
|
130
|
-
|
|
138
|
+
images: np.ndarray,
|
|
131
139
|
start: int = None,
|
|
132
|
-
stop: int = None
|
|
133
|
-
|
|
140
|
+
stop: int = None,
|
|
141
|
+
bg_off: np.ndarray = None,
|
|
142
|
+
):
|
|
143
|
+
"""Perform batch segmentation of `images`
|
|
144
|
+
|
|
145
|
+
Before segmentation, an optional background offset correction with
|
|
146
|
+
`bg_off` is performed. After segmentation, mask postprocessing is
|
|
147
|
+
performed according to the class definition.
|
|
134
148
|
|
|
135
149
|
Parameters
|
|
136
150
|
----------
|
|
137
|
-
|
|
151
|
+
images: 3d np.ndarray
|
|
138
152
|
The time-series image data. First axis is time.
|
|
139
153
|
start: int
|
|
140
|
-
First index to analyze in `
|
|
154
|
+
First index to analyze in `images`
|
|
141
155
|
stop: int
|
|
142
|
-
Index after the last index to analyze in `
|
|
156
|
+
Index after the last index to analyze in `images`
|
|
157
|
+
bg_off: 1D np.ndarray
|
|
158
|
+
Optional 1D numpy array with background offset
|
|
143
159
|
|
|
144
160
|
Notes
|
|
145
161
|
-----
|
|
146
162
|
- If the segmentation algorithm only accepts background-corrected
|
|
147
|
-
images, then `
|
|
163
|
+
images, then `images` must already be background-corrected,
|
|
164
|
+
except for the optional `bg_off`.
|
|
148
165
|
"""
|
|
149
166
|
if stop is None or start is None:
|
|
150
167
|
start = 0
|
|
151
|
-
stop = len(
|
|
168
|
+
stop = len(images)
|
|
152
169
|
|
|
153
170
|
batch_size = stop - start
|
|
154
|
-
size = np.prod(
|
|
171
|
+
size = np.prod(images.shape[1:]) * batch_size
|
|
155
172
|
|
|
156
173
|
if self.image_shape is None:
|
|
157
|
-
self.image_shape =
|
|
174
|
+
self.image_shape = images[0].shape
|
|
158
175
|
|
|
159
176
|
if self._mp_image_np is not None and self._mp_image_np.size != size:
|
|
160
177
|
# reset image data
|
|
161
178
|
self._mp_image_np = None
|
|
162
179
|
self._mp_labels_np = None
|
|
180
|
+
self._mp_bg_off_np = None
|
|
163
181
|
# TODO: If only the batch_size changes, don't
|
|
164
182
|
# reinitialize the workers. Otherwise, the final rest of
|
|
165
183
|
# analyzing a dataset would always take a little longer.
|
|
@@ -168,31 +186,47 @@ class CPUSegmenter(Segmenter, abc.ABC):
|
|
|
168
186
|
self.mp_batch_index.value = -1
|
|
169
187
|
self.mp_shutdown.value = 0
|
|
170
188
|
|
|
189
|
+
if bg_off is not None:
|
|
190
|
+
if not self.requires_background_correction:
|
|
191
|
+
raise ValueError(f"The segmenter {self.__class__.__name__} "
|
|
192
|
+
f"does not employ background correction, "
|
|
193
|
+
f"but the `bg_off` keyword argument was "
|
|
194
|
+
f"passed to `segment_chunk`. Please check "
|
|
195
|
+
f"your analysis pipeline.")
|
|
196
|
+
# background offset
|
|
197
|
+
if self._mp_bg_off_np is None:
|
|
198
|
+
self.mp_bg_off_raw, self._mp_bg_off_np = \
|
|
199
|
+
self._create_shared_array(
|
|
200
|
+
array_shape=(stop - start,),
|
|
201
|
+
batch_size=batch_size,
|
|
202
|
+
dtype=np.float64)
|
|
203
|
+
self._mp_bg_off_np[:] = bg_off[start:stop]
|
|
204
|
+
|
|
205
|
+
# input images
|
|
171
206
|
if self._mp_image_np is None:
|
|
172
207
|
self.mp_image_raw, self._mp_image_np = self._create_shared_array(
|
|
173
|
-
|
|
208
|
+
array_shape=self.image_shape,
|
|
174
209
|
batch_size=batch_size,
|
|
175
|
-
dtype=
|
|
210
|
+
dtype=images.dtype,
|
|
176
211
|
)
|
|
212
|
+
self._mp_image_np[:] = images[start:stop]
|
|
177
213
|
|
|
214
|
+
# output labels
|
|
178
215
|
if self._mp_labels_np is None:
|
|
179
216
|
self.mp_labels_raw, self._mp_labels_np = self._create_shared_array(
|
|
180
|
-
|
|
217
|
+
array_shape=self.image_shape,
|
|
181
218
|
batch_size=batch_size,
|
|
182
219
|
dtype=np.uint16,
|
|
183
220
|
)
|
|
184
221
|
|
|
185
|
-
# populate image data
|
|
186
|
-
self._mp_image_np[:] = image_data[start:stop]
|
|
187
|
-
|
|
188
222
|
# Create the workers
|
|
189
223
|
if self.debug:
|
|
190
|
-
worker_cls =
|
|
224
|
+
worker_cls = MPOSegmenterWorkerThread
|
|
191
225
|
num_workers = 1
|
|
192
226
|
self.logger.debug("Running with one worker in main thread")
|
|
193
227
|
else:
|
|
194
|
-
worker_cls =
|
|
195
|
-
num_workers = min(self.num_workers,
|
|
228
|
+
worker_cls = MPOSegmenterWorkerProcess
|
|
229
|
+
num_workers = min(self.num_workers, images.shape[0])
|
|
196
230
|
self.logger.debug(f"Running with {num_workers} workers")
|
|
197
231
|
|
|
198
232
|
if not self._mp_workers:
|
|
@@ -224,8 +258,33 @@ class CPUSegmenter(Segmenter, abc.ABC):
|
|
|
224
258
|
|
|
225
259
|
return self._mp_labels_np
|
|
226
260
|
|
|
261
|
+
def segment_single(self, image, bg_off: float = None):
|
|
262
|
+
"""Return the integer label image for an input image
|
|
263
|
+
|
|
264
|
+
Before segmentation, an optional background offset correction with
|
|
265
|
+
`bg_off` is performed. After segmentation, mask postprocessing is
|
|
266
|
+
performed according to the class definition.
|
|
267
|
+
"""
|
|
268
|
+
segm_wrap = self.segment_algorithm_wrapper()
|
|
269
|
+
# optional subtraction of background offset
|
|
270
|
+
if bg_off is not None:
|
|
271
|
+
image = image - bg_off
|
|
272
|
+
# obtain mask or label
|
|
273
|
+
mol = segm_wrap(image)
|
|
274
|
+
if mol.dtype == bool:
|
|
275
|
+
# convert mask to labels
|
|
276
|
+
labels, _ = ndi.label(
|
|
277
|
+
input=mol,
|
|
278
|
+
structure=ndi.generate_binary_structure(2, 2))
|
|
279
|
+
else:
|
|
280
|
+
labels = mol
|
|
281
|
+
# optional mask/label postprocessing
|
|
282
|
+
if self.mask_postprocessing:
|
|
283
|
+
labels = self.process_mask(labels, **self.kwargs_mask)
|
|
284
|
+
return labels
|
|
285
|
+
|
|
227
286
|
|
|
228
|
-
class
|
|
287
|
+
class MPOSegmenterWorker:
|
|
229
288
|
def __init__(self,
|
|
230
289
|
segmenter,
|
|
231
290
|
sl_start: int,
|
|
@@ -235,7 +294,7 @@ class CPUSegmenterWorker:
|
|
|
235
294
|
|
|
236
295
|
Parameters
|
|
237
296
|
----------
|
|
238
|
-
segmenter:
|
|
297
|
+
segmenter: MPOSegmenter
|
|
239
298
|
The segmentation instance
|
|
240
299
|
sl_start: int
|
|
241
300
|
Start of slice of input array to process
|
|
@@ -243,7 +302,7 @@ class CPUSegmenterWorker:
|
|
|
243
302
|
Stop of slice of input array to process
|
|
244
303
|
"""
|
|
245
304
|
# Must call super init, otherwise Thread or Process are not initialized
|
|
246
|
-
super(
|
|
305
|
+
super(MPOSegmenterWorker, self).__init__()
|
|
247
306
|
self.segmenter = segmenter
|
|
248
307
|
# Value incrementing the batch index. Starts with 0 and is
|
|
249
308
|
# incremented every time :func:`Segmenter.segment_batch` is
|
|
@@ -255,8 +314,10 @@ class CPUSegmenterWorker:
|
|
|
255
314
|
# Shutdown bit tells workers to stop when set to != 0
|
|
256
315
|
self.shutdown = segmenter.mp_shutdown
|
|
257
316
|
# The image data for segmentation
|
|
258
|
-
self.
|
|
259
|
-
#
|
|
317
|
+
self.image_arr_raw = segmenter.mp_image_raw
|
|
318
|
+
# Background data offset
|
|
319
|
+
self.bg_off = segmenter.mp_bg_off_raw
|
|
320
|
+
# Integer output label array
|
|
260
321
|
self.labels_data_raw = segmenter.mp_labels_raw
|
|
261
322
|
# The shape of one image
|
|
262
323
|
self.image_shape = segmenter.image_shape
|
|
@@ -268,10 +329,14 @@ class CPUSegmenterWorker:
|
|
|
268
329
|
# We have to create the numpy-versions of the mp.RawArrays here,
|
|
269
330
|
# otherwise we only get some kind of copy in the new process
|
|
270
331
|
# when we use "spawn" instead of "fork".
|
|
271
|
-
|
|
332
|
+
labels_arr = np.ctypeslib.as_array(self.labels_data_raw).reshape(
|
|
272
333
|
-1, self.image_shape[0], self.image_shape[1])
|
|
273
|
-
|
|
334
|
+
image_arr = np.ctypeslib.as_array(self.image_arr_raw).reshape(
|
|
274
335
|
-1, self.image_shape[0], self.image_shape[1])
|
|
336
|
+
if self.bg_off is not None:
|
|
337
|
+
bg_off_data = np.ctypeslib.as_array(self.bg_off)
|
|
338
|
+
else:
|
|
339
|
+
bg_off_data = None
|
|
275
340
|
|
|
276
341
|
idx = self.sl_start
|
|
277
342
|
itr = 0 # current iteration (incremented when we reach self.sl_stop)
|
|
@@ -285,8 +350,13 @@ class CPUSegmenterWorker:
|
|
|
285
350
|
with self.batch_worker:
|
|
286
351
|
self.batch_worker.value += 1
|
|
287
352
|
else:
|
|
288
|
-
|
|
289
|
-
|
|
353
|
+
if bg_off_data is None:
|
|
354
|
+
bg_off = None
|
|
355
|
+
else:
|
|
356
|
+
bg_off = bg_off_data[idx]
|
|
357
|
+
|
|
358
|
+
labels_arr[idx, :, :] = self.segmenter.segment_single(
|
|
359
|
+
image=image_arr[idx], bg_off=bg_off)
|
|
290
360
|
idx += 1
|
|
291
361
|
elif self.shutdown.value:
|
|
292
362
|
break
|
|
@@ -295,11 +365,11 @@ class CPUSegmenterWorker:
|
|
|
295
365
|
time.sleep(.01)
|
|
296
366
|
|
|
297
367
|
|
|
298
|
-
class
|
|
368
|
+
class MPOSegmenterWorkerProcess(MPOSegmenterWorker, mp_spawn.Process):
|
|
299
369
|
def __init__(self, *args, **kwargs):
|
|
300
|
-
super(
|
|
370
|
+
super(MPOSegmenterWorkerProcess, self).__init__(*args, **kwargs)
|
|
301
371
|
|
|
302
372
|
|
|
303
|
-
class
|
|
373
|
+
class MPOSegmenterWorkerThread(MPOSegmenterWorker, threading.Thread):
|
|
304
374
|
def __init__(self, *args, **kwargs):
|
|
305
|
-
super(
|
|
375
|
+
super(MPOSegmenterWorkerThread, self).__init__(*args, **kwargs)
|