dcnum 0.16.5__py3-none-any.whl → 0.16.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dcnum might be problematic. Click here for more details.
- dcnum/_version.py +2 -2
- dcnum/feat/event_extractor_manager_thread.py +16 -0
- dcnum/feat/feat_background/bg_roll_median.py +8 -8
- dcnum/feat/feat_background/bg_sparse_median.py +7 -7
- dcnum/feat/gate.py +107 -99
- dcnum/logic/ctrl.py +1 -0
- dcnum/read/hdf5_data.py +4 -3
- dcnum/segm/segmenter.py +17 -7
- dcnum/write/deque_writer_thread.py +5 -3
- dcnum/write/writer.py +6 -6
- {dcnum-0.16.5.dist-info → dcnum-0.16.7.dist-info}/METADATA +2 -2
- {dcnum-0.16.5.dist-info → dcnum-0.16.7.dist-info}/RECORD +15 -15
- {dcnum-0.16.5.dist-info → dcnum-0.16.7.dist-info}/LICENSE +0 -0
- {dcnum-0.16.5.dist-info → dcnum-0.16.7.dist-info}/WHEEL +0 -0
- {dcnum-0.16.5.dist-info → dcnum-0.16.7.dist-info}/top_level.txt +0 -0
dcnum/_version.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Feature computation: managing event extraction threads"""
|
|
2
|
+
import collections
|
|
2
3
|
import logging
|
|
3
4
|
import multiprocessing as mp
|
|
4
5
|
import threading
|
|
@@ -17,6 +18,7 @@ class EventExtractorManagerThread(threading.Thread):
|
|
|
17
18
|
labels_list: List,
|
|
18
19
|
fe_kwargs: Dict,
|
|
19
20
|
num_workers: int,
|
|
21
|
+
writer_dq: collections.deque,
|
|
20
22
|
debug: bool = False,
|
|
21
23
|
*args, **kwargs):
|
|
22
24
|
"""Manage event extraction threads or precesses
|
|
@@ -40,6 +42,9 @@ class EventExtractorManagerThread(threading.Thread):
|
|
|
40
42
|
:func:`.EventExtractor.get_init_kwargs` for more information.
|
|
41
43
|
num_workers:
|
|
42
44
|
Number of child threads or worker processes to use.
|
|
45
|
+
writer_dq:
|
|
46
|
+
The queue the writer uses. We monitor this queue. If it
|
|
47
|
+
fills up, we take a break.
|
|
43
48
|
debug:
|
|
44
49
|
Whether to run in debugging mode which means more log
|
|
45
50
|
messages and only one thread (`num_workers` has no effect).
|
|
@@ -66,6 +71,8 @@ class EventExtractorManagerThread(threading.Thread):
|
|
|
66
71
|
self.label_array = np.ctypeslib.as_array(
|
|
67
72
|
self.fe_kwargs["label_array"]).reshape(
|
|
68
73
|
self.data.image.chunk_shape)
|
|
74
|
+
#: Writer deque to monitor
|
|
75
|
+
self.writer_dq = writer_dq
|
|
69
76
|
#: Time counter for feature extraction
|
|
70
77
|
self.t_count = 0
|
|
71
78
|
#: Whether debugging is enabled
|
|
@@ -86,6 +93,15 @@ class EventExtractorManagerThread(threading.Thread):
|
|
|
86
93
|
chunks_processed = 0
|
|
87
94
|
frames_processed = 0
|
|
88
95
|
while True:
|
|
96
|
+
# If the writer_dq starts filling up, then this could lead to
|
|
97
|
+
# an oom-kill signal. Stall for the writer to prevent this.
|
|
98
|
+
ldq = len(self.writer_dq)
|
|
99
|
+
if ldq > 100:
|
|
100
|
+
stallsec = ldq / 100
|
|
101
|
+
self.logger.warning(
|
|
102
|
+
f"Stalling {stallsec:.1f}s for slow writer.")
|
|
103
|
+
time.sleep(stallsec)
|
|
104
|
+
|
|
89
105
|
cur_slot = 0
|
|
90
106
|
unavailable_slots = 0
|
|
91
107
|
# Check all slots for segmented labels
|
|
@@ -95,12 +95,12 @@ class BackgroundRollMed(Background):
|
|
|
95
95
|
#: queue for median computation jobs
|
|
96
96
|
self.queue = mp_spawn.Queue()
|
|
97
97
|
#: list of workers (processes)
|
|
98
|
-
self.workers = [
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
98
|
+
self.workers = [WorkerRollMed(self.queue,
|
|
99
|
+
self.worker_counter,
|
|
100
|
+
self.shared_input_raw,
|
|
101
|
+
self.shared_output_raw,
|
|
102
|
+
self.batch_size,
|
|
103
|
+
self.kernel_size)
|
|
104
104
|
for _ in range(self.num_cpus)]
|
|
105
105
|
[w.start() for w in self.workers]
|
|
106
106
|
|
|
@@ -216,11 +216,11 @@ class BackgroundRollMed(Background):
|
|
|
216
216
|
self.image_proc.value += self.batch_size
|
|
217
217
|
|
|
218
218
|
|
|
219
|
-
class
|
|
219
|
+
class WorkerRollMed(mp_spawn.Process):
|
|
220
220
|
def __init__(self, job_queue, counter, shared_input, shared_output,
|
|
221
221
|
batch_size, kernel_size, *args, **kwargs):
|
|
222
222
|
"""Worker process for median computation"""
|
|
223
|
-
super(
|
|
223
|
+
super(WorkerRollMed, self).__init__(*args, **kwargs)
|
|
224
224
|
self.queue = job_queue
|
|
225
225
|
self.queue.cancel_join_thread()
|
|
226
226
|
self.counter = counter
|
|
@@ -152,11 +152,11 @@ class BackgroundSparseMed(Background):
|
|
|
152
152
|
#: queue for median computation jobs
|
|
153
153
|
self.queue = mp_spawn.Queue()
|
|
154
154
|
#: list of workers (processes)
|
|
155
|
-
self.workers = [
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
155
|
+
self.workers = [WorkerSparseMed(self.queue,
|
|
156
|
+
self.worker_counter,
|
|
157
|
+
self.shared_input_raw,
|
|
158
|
+
self.shared_output_raw,
|
|
159
|
+
self.kernel_size)
|
|
160
160
|
for _ in range(self.num_cpus)]
|
|
161
161
|
[w.start() for w in self.workers]
|
|
162
162
|
|
|
@@ -348,11 +348,11 @@ class BackgroundSparseMed(Background):
|
|
|
348
348
|
self.image_proc.value = idx_stop
|
|
349
349
|
|
|
350
350
|
|
|
351
|
-
class
|
|
351
|
+
class WorkerSparseMed(mp_spawn.Process):
|
|
352
352
|
def __init__(self, job_queue, counter, shared_input, shared_output,
|
|
353
353
|
kernel_size, *args, **kwargs):
|
|
354
354
|
"""Worker process for median computation"""
|
|
355
|
-
super(
|
|
355
|
+
super(WorkerSparseMed, self).__init__(*args, **kwargs)
|
|
356
356
|
self.queue = job_queue
|
|
357
357
|
self.queue.cancel_join_thread()
|
|
358
358
|
self.counter = counter
|
dcnum/feat/gate.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
"""Feature
|
|
1
|
+
"""Feature gating"""
|
|
2
2
|
import copy
|
|
3
|
+
import numbers
|
|
3
4
|
import warnings
|
|
4
5
|
|
|
5
6
|
import numpy as np
|
|
@@ -11,7 +12,8 @@ class Gate:
|
|
|
11
12
|
#: the default value for `size_thresh_mask` if not given as kwarg
|
|
12
13
|
_default_size_thresh_mask = 10
|
|
13
14
|
|
|
14
|
-
def __init__(self, data, *,
|
|
15
|
+
def __init__(self, data, *,
|
|
16
|
+
online_gates: bool = False,
|
|
15
17
|
size_thresh_mask: int = None):
|
|
16
18
|
"""Gate feature data
|
|
17
19
|
|
|
@@ -20,51 +22,43 @@ class Gate:
|
|
|
20
22
|
data: .HDF5Data
|
|
21
23
|
dcevent data instance
|
|
22
24
|
online_gates: bool
|
|
23
|
-
set to True to enable gating with
|
|
25
|
+
set to True to enable gating with "online" gates stored
|
|
26
|
+
in the input file; online gates are applied in real-time
|
|
27
|
+
deformability cytometry before writing data to disk during
|
|
28
|
+
a measurement
|
|
24
29
|
size_thresh_mask: int
|
|
25
30
|
Only masks with more pixels than `size_thresh_mask` are
|
|
26
|
-
considered to be a valid event; Originally,
|
|
27
|
-
`trig_thresh` value defaulted to 200
|
|
28
|
-
|
|
31
|
+
considered to be a valid event; Originally, the
|
|
32
|
+
`bin area min`/`trig_thresh` value defaulted to 200 which is
|
|
33
|
+
too large; defaults to 10 or the original value in case
|
|
34
|
+
`online_gates` is set.
|
|
29
35
|
"""
|
|
30
|
-
#:
|
|
31
|
-
self.
|
|
32
|
-
|
|
33
|
-
#: gating keyword arguments
|
|
34
|
-
self.kwargs = {}
|
|
36
|
+
#: box gating (value range for each feature)
|
|
37
|
+
self.box_gates = {}
|
|
35
38
|
|
|
36
39
|
if online_gates:
|
|
37
|
-
|
|
38
|
-
#
|
|
39
|
-
self.
|
|
40
|
-
|
|
41
|
-
#
|
|
42
|
-
# original file, then set the default value.
|
|
43
|
-
if self.kwargs.get("size_thresh_mask") is None:
|
|
44
|
-
self.kwargs["size_thresh_mask"] = \
|
|
45
|
-
self._default_size_thresh_mask
|
|
46
|
-
else:
|
|
47
|
-
self.box_gates = {}
|
|
48
|
-
# If the user did not provide a size_thresh_mask, use the default.
|
|
40
|
+
# Deal with online gates.
|
|
41
|
+
# First, compute the box gates.
|
|
42
|
+
self.box_gates.update(self._extract_online_gates(data))
|
|
43
|
+
# If the user did not specify a threshold, attempt to extract
|
|
44
|
+
# it from the metadata.
|
|
49
45
|
if size_thresh_mask is None:
|
|
50
|
-
size_thresh_mask =
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
self.kwargs["online_gates"] = online_gates
|
|
54
|
-
|
|
55
|
-
def _set_kwarg(self, name, sec, key, user_value):
|
|
56
|
-
if user_value is None:
|
|
57
|
-
value = self.data.meta_nest.get(sec, {}).get(key)
|
|
58
|
-
else:
|
|
59
|
-
value = user_value
|
|
60
|
-
if value is not None:
|
|
61
|
-
self.kwargs[name] = value
|
|
46
|
+
size_thresh_mask = data.meta_nest.get(
|
|
47
|
+
"online_contour", {}).get("bin area min")
|
|
62
48
|
|
|
63
|
-
|
|
64
|
-
|
|
49
|
+
#: gating keyword arguments
|
|
50
|
+
self.kwargs = {
|
|
51
|
+
"online_gates": online_gates,
|
|
52
|
+
# Set the size threshold, defaulting to `_default_size_thresh_mask`
|
|
53
|
+
"size_thresh_mask":
|
|
54
|
+
size_thresh_mask or self._default_size_thresh_mask
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
def _extract_online_gates(self, data):
|
|
58
|
+
ogates = {}
|
|
65
59
|
# Extract online filters from the dataset
|
|
66
|
-
|
|
67
|
-
for key in
|
|
60
|
+
source_meta = data.meta_nest.get("online_filter", {})
|
|
61
|
+
for key in source_meta:
|
|
68
62
|
if key.endswith("polygon points"):
|
|
69
63
|
raise NotImplementedError("Polygon gating not implemented!")
|
|
70
64
|
elif (key.endswith("soft limit")
|
|
@@ -72,66 +66,30 @@ class Gate:
|
|
|
72
66
|
# we only want hard gates
|
|
73
67
|
continue
|
|
74
68
|
else:
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
69
|
+
try:
|
|
70
|
+
feat, lim = key.rsplit(' ', 1)
|
|
71
|
+
lim_idx = ["min", "max"].index(lim)
|
|
72
|
+
except ValueError:
|
|
73
|
+
warnings.warn(f"Unexpected online gate '{key}'")
|
|
74
|
+
else:
|
|
75
|
+
# make sure we are not dealing with a soft limit
|
|
76
|
+
if not source_meta.get(f"{feat} soft limit", True):
|
|
77
|
+
ogates.setdefault(feat, [None, None])
|
|
78
|
+
ogates[feat][lim_idx] = source_meta[key]
|
|
79
79
|
|
|
80
80
|
# This is somehow hard-coded in Shape-In (minimum size is 3px)
|
|
81
|
-
px_size =
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
return all_online_filters
|
|
88
|
-
|
|
89
|
-
def gate_feature(self, feat, data):
|
|
90
|
-
valid_left = True
|
|
91
|
-
valid_right = True
|
|
92
|
-
if f"{feat} min" in self.box_gates:
|
|
93
|
-
valid_left = data > self.box_gates[f"{feat} min"]
|
|
94
|
-
if f"{feat} max" in self.box_gates:
|
|
95
|
-
valid_right = data < self.box_gates[f"{feat} max"]
|
|
96
|
-
return np.logical_and(valid_left, valid_right)
|
|
81
|
+
px_size = data.pixel_size
|
|
82
|
+
ogates["size_x"] = [
|
|
83
|
+
max(ogates.get("size_x min", 0), 3 * px_size), None]
|
|
84
|
+
ogates["size_y"] = [
|
|
85
|
+
max(ogates.get("size_y min", 0), 3 * px_size), None]
|
|
97
86
|
|
|
98
|
-
|
|
99
|
-
"""Return None if the event should not be used, else `event`"""
|
|
100
|
-
if self.box_gates and event:
|
|
101
|
-
# Only use those events that are within the limits of the
|
|
102
|
-
# online filters.
|
|
103
|
-
for feat in self.features:
|
|
104
|
-
if not self.gate_feature(feat, event[feat]):
|
|
105
|
-
return
|
|
106
|
-
return event
|
|
87
|
+
return ogates
|
|
107
88
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
size = len(events[key0])
|
|
113
|
-
valid = np.ones(size, dtype=bool)
|
|
114
|
-
for feat in self.features:
|
|
115
|
-
valid = np.logical_and(valid,
|
|
116
|
-
self.gate_feature(feat, events[feat])
|
|
117
|
-
)
|
|
118
|
-
else:
|
|
119
|
-
raise ValueError("Empty events provided!")
|
|
120
|
-
return valid
|
|
121
|
-
|
|
122
|
-
def gate_mask(self, mask, mask_sum=None):
|
|
123
|
-
"""Gate the mask, return False if the mask should not be used
|
|
124
|
-
|
|
125
|
-
Parameters
|
|
126
|
-
----------
|
|
127
|
-
mask: 2d ndarray
|
|
128
|
-
The boolean mask image for the event.
|
|
129
|
-
mask_sum: int
|
|
130
|
-
The sum of the mask (if not specified, it is computed)
|
|
131
|
-
"""
|
|
132
|
-
if mask_sum is None:
|
|
133
|
-
mask_sum = np.sum(mask)
|
|
134
|
-
return mask_sum > self.kwargs["size_thresh_mask"]
|
|
89
|
+
@property
|
|
90
|
+
def features(self):
|
|
91
|
+
"""Sorted list of feature gates defined"""
|
|
92
|
+
return sorted(self.box_gates.keys())
|
|
135
93
|
|
|
136
94
|
def get_ppid(self):
|
|
137
95
|
"""Return a unique gating pipeline identifier
|
|
@@ -181,13 +139,63 @@ class Gate:
|
|
|
181
139
|
ppid=pp_gate_kwargs)
|
|
182
140
|
return kwargs
|
|
183
141
|
|
|
184
|
-
@property
|
|
185
|
-
def features(self):
|
|
186
|
-
return [kk.split()[0] for kk in list(self.box_gates.keys())]
|
|
187
|
-
|
|
188
142
|
@classmethod
|
|
189
143
|
def get_ppid_from_kwargs(cls, kwargs):
|
|
190
144
|
warnings.warn(
|
|
191
145
|
"Please use get_ppid_from_ppkw instead of get_ppid_from_kwargs",
|
|
192
146
|
DeprecationWarning)
|
|
193
147
|
return cls.get_ppid_from_ppkw(kwargs)
|
|
148
|
+
|
|
149
|
+
def gate_event(self, event):
|
|
150
|
+
"""Return None if the event should not be used, else `event`"""
|
|
151
|
+
if self.box_gates and event:
|
|
152
|
+
# Only use those events that are within the limits of the
|
|
153
|
+
# online filters.
|
|
154
|
+
for feat in self.features:
|
|
155
|
+
if not self.gate_feature(feat, event[feat]):
|
|
156
|
+
return
|
|
157
|
+
return event
|
|
158
|
+
|
|
159
|
+
def gate_events(self, events):
|
|
160
|
+
"""Return boolean array with events that should be used"""
|
|
161
|
+
if self.box_gates and bool(events):
|
|
162
|
+
key0 = list(events.keys())[0]
|
|
163
|
+
size = len(events[key0])
|
|
164
|
+
valid = np.ones(size, dtype=bool)
|
|
165
|
+
for feat in self.features:
|
|
166
|
+
valid = np.logical_and(valid,
|
|
167
|
+
self.gate_feature(feat, events[feat])
|
|
168
|
+
)
|
|
169
|
+
else:
|
|
170
|
+
raise ValueError("Empty events provided!")
|
|
171
|
+
return valid
|
|
172
|
+
|
|
173
|
+
def gate_feature(self,
|
|
174
|
+
feat: str,
|
|
175
|
+
data: numbers.Number | np.ndarray):
|
|
176
|
+
"""Return boolean indicating whether `data` value is in box gate
|
|
177
|
+
|
|
178
|
+
`data` may be a number or an array. If no box filter is defined
|
|
179
|
+
for `feat`, `True` is always returned. Otherwise, either a boolean
|
|
180
|
+
or a boolean array is returned, depending on the type of `data`.
|
|
181
|
+
Not that `np.logical_and` can deal with mixed argument types
|
|
182
|
+
(scalar and array).
|
|
183
|
+
"""
|
|
184
|
+
bound_lo, bound_up = self.box_gates[feat]
|
|
185
|
+
valid_lo = data >= bound_lo if bound_lo is not None else True
|
|
186
|
+
valid_up = data <= bound_up if bound_up is not None else True
|
|
187
|
+
return np.logical_and(valid_lo, valid_up)
|
|
188
|
+
|
|
189
|
+
def gate_mask(self, mask, mask_sum=None):
|
|
190
|
+
"""Gate the mask, return False if the mask should not be used
|
|
191
|
+
|
|
192
|
+
Parameters
|
|
193
|
+
----------
|
|
194
|
+
mask: 2d ndarray
|
|
195
|
+
The boolean mask image for the event.
|
|
196
|
+
mask_sum: int
|
|
197
|
+
The sum of the mask (if not specified, it is computed)
|
|
198
|
+
"""
|
|
199
|
+
if mask_sum is None:
|
|
200
|
+
mask_sum = np.sum(mask)
|
|
201
|
+
return mask_sum > self.kwargs["size_thresh_mask"]
|
dcnum/logic/ctrl.py
CHANGED
dcnum/read/hdf5_data.py
CHANGED
|
@@ -386,8 +386,8 @@ def concatenated_hdf5_data(paths: List[pathlib.Path],
|
|
|
386
386
|
path_out:
|
|
387
387
|
If `None`, then the dataset is created in memory. If `True`
|
|
388
388
|
(default), create a file on disk. If a pathlib.Path is specified,
|
|
389
|
-
the dataset is written to that file. Note that
|
|
390
|
-
are
|
|
389
|
+
the dataset is written to that file. Note that datasets in memory
|
|
390
|
+
are likely not pickable (so don't use them for multiprocessing).
|
|
391
391
|
compute_frame:
|
|
392
392
|
Whether to compute the "events/frame" feature, taking the frame
|
|
393
393
|
data from the input files and properly incrementing them along
|
|
@@ -435,7 +435,8 @@ def concatenated_hdf5_data(paths: List[pathlib.Path],
|
|
|
435
435
|
# get metadata
|
|
436
436
|
if ii == 0:
|
|
437
437
|
meta = dict(h5.attrs)
|
|
438
|
-
|
|
438
|
+
if not features:
|
|
439
|
+
features = featsi
|
|
439
440
|
# make sure number of features are consistent
|
|
440
441
|
if not set(features) <= set(featsi):
|
|
441
442
|
raise ValueError(
|
dcnum/segm/segmenter.py
CHANGED
|
@@ -64,6 +64,17 @@ class Segmenter(abc.ABC):
|
|
|
64
64
|
"`kwargs_mask` has been specified, but mask post-processing "
|
|
65
65
|
f"is disabled for segmenter {self.__class__}")
|
|
66
66
|
|
|
67
|
+
@staticmethod
|
|
68
|
+
@functools.cache
|
|
69
|
+
def get_border(shape):
|
|
70
|
+
"""Cached boolean image with outer pixels set to True"""
|
|
71
|
+
border = np.zeros(shape, dtype=bool)
|
|
72
|
+
border[0, :] = True
|
|
73
|
+
border[-1, :] = True
|
|
74
|
+
border[:, 0] = True
|
|
75
|
+
border[:, -1] = True
|
|
76
|
+
return border
|
|
77
|
+
|
|
67
78
|
@staticmethod
|
|
68
79
|
@functools.cache
|
|
69
80
|
def get_disk(radius):
|
|
@@ -178,14 +189,13 @@ class Segmenter(abc.ABC):
|
|
|
178
189
|
#
|
|
179
190
|
if (labels[0, :].sum() or labels[-1, :].sum()
|
|
180
191
|
or labels[:, 0].sum() or labels[:, -1].sum()):
|
|
181
|
-
border =
|
|
182
|
-
border[0] = True
|
|
183
|
-
border[-1] = True
|
|
184
|
-
border[:, 0] = True
|
|
185
|
-
border[:, -1] = True
|
|
192
|
+
border = Segmenter.get_border(labels.shape)
|
|
186
193
|
indices = sorted(np.unique(labels[border]))
|
|
187
|
-
for
|
|
188
|
-
|
|
194
|
+
for li in indices:
|
|
195
|
+
if li == 0:
|
|
196
|
+
# ignore background values
|
|
197
|
+
continue
|
|
198
|
+
labels[labels == li] = 0
|
|
189
199
|
|
|
190
200
|
# scikit-image is too slow for us here. So we use OpenCV.
|
|
191
201
|
# https://github.com/scikit-image/scikit-image/issues/1190
|
|
@@ -41,11 +41,13 @@ class DequeWriterThread(threading.Thread):
|
|
|
41
41
|
|
|
42
42
|
def run(self):
|
|
43
43
|
while True:
|
|
44
|
+
ldq = len(self.dq)
|
|
44
45
|
if self.must_stop_loop:
|
|
45
46
|
break
|
|
46
|
-
elif
|
|
47
|
-
|
|
48
|
-
|
|
47
|
+
elif ldq:
|
|
48
|
+
for _ in range(ldq):
|
|
49
|
+
feat, data = self.dq.popleft()
|
|
50
|
+
self.writer.store_feature_chunk(feat=feat, data=data)
|
|
49
51
|
elif self.may_stop_loop:
|
|
50
52
|
break
|
|
51
53
|
else:
|
dcnum/write/writer.py
CHANGED
|
@@ -35,7 +35,7 @@ class HDF5Writer:
|
|
|
35
35
|
|
|
36
36
|
@staticmethod
|
|
37
37
|
def get_best_nd_chunks(item_shape, feat_dtype=np.float64):
|
|
38
|
-
"""Return best chunks for
|
|
38
|
+
"""Return best chunks for HDF5 datasets
|
|
39
39
|
|
|
40
40
|
Chunking has performance implications. It’s recommended to keep the
|
|
41
41
|
total size of dataset chunks between 10 KiB and 1 MiB. This number
|
|
@@ -44,6 +44,7 @@ class HDF5Writer:
|
|
|
44
44
|
"""
|
|
45
45
|
# set image feature chunk size to approximately 1MiB
|
|
46
46
|
num_bytes = 1024 ** 2
|
|
47
|
+
# Note that `np.prod(()) == 1`
|
|
47
48
|
event_size = np.prod(item_shape) * np.dtype(feat_dtype).itemsize
|
|
48
49
|
chunk_size = num_bytes / event_size
|
|
49
50
|
# Set minimum chunk size to 10 so that we can have at least some
|
|
@@ -53,12 +54,11 @@ class HDF5Writer:
|
|
|
53
54
|
|
|
54
55
|
def require_feature(self, feat, item_shape, feat_dtype, ds_kwds=None):
|
|
55
56
|
"""Create a new feature in the "events" group"""
|
|
56
|
-
|
|
57
|
-
if ds_kwds is None:
|
|
58
|
-
ds_kwds = {}
|
|
59
|
-
for key in self.ds_kwds:
|
|
60
|
-
ds_kwds.setdefault(key, self.ds_kwds[key])
|
|
61
57
|
if feat not in self.events:
|
|
58
|
+
if ds_kwds is None:
|
|
59
|
+
ds_kwds = {}
|
|
60
|
+
for key in self.ds_kwds:
|
|
61
|
+
ds_kwds.setdefault(key, self.ds_kwds[key])
|
|
62
62
|
dset = self.events.create_dataset(
|
|
63
63
|
feat,
|
|
64
64
|
shape=tuple([0] + list(item_shape)),
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dcnum
|
|
3
|
-
Version: 0.16.
|
|
3
|
+
Version: 0.16.7
|
|
4
4
|
Summary: numerics toolbox for imaging deformability cytometry
|
|
5
|
-
Author: Paul Müller
|
|
5
|
+
Author: Maximilian Schlögel, Paul Müller
|
|
6
6
|
Maintainer-email: Paul Müller <dev@craban.de>
|
|
7
7
|
License: MIT
|
|
8
8
|
Project-URL: source, https://github.com/DC-Analysis/dcnum
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
dcnum/__init__.py,sha256=hcawIKS7utYiOyVhOAX9t7K3xYzP1b9862VV0b6qSrQ,74
|
|
2
|
-
dcnum/_version.py,sha256=
|
|
2
|
+
dcnum/_version.py,sha256=qnrnqIwaxxSmbCyOa5NQksqjVQLJBzYOk8VUPMYeuoY,413
|
|
3
3
|
dcnum/feat/__init__.py,sha256=JqlgzOgDJhoTk8WVYcIiKTWq9EAM16_jGivzOtN6JGo,325
|
|
4
|
-
dcnum/feat/event_extractor_manager_thread.py,sha256=
|
|
5
|
-
dcnum/feat/gate.py,sha256=
|
|
4
|
+
dcnum/feat/event_extractor_manager_thread.py,sha256=NSoey-h-STIAvadHWDbkhXsB6FZ_SkYnBbEC6JWkTQ0,7130
|
|
5
|
+
dcnum/feat/gate.py,sha256=ZTV2tkNqJBGmwemZrI0IAaPXTLgWO_5ZVxnpqr2G1cI,7464
|
|
6
6
|
dcnum/feat/queue_event_extractor.py,sha256=eacmcM_uHHW6GtqCCtDCZib9423TGvuqgxGKAxv0tgc,15686
|
|
7
7
|
dcnum/feat/feat_background/__init__.py,sha256=OTmMuazHNaSrZb2XW4cnJ6PlgJLbKrPbaidpEixYa0A,341
|
|
8
8
|
dcnum/feat/feat_background/base.py,sha256=IPHJN9Stdk7WZbbAn5VPNLMZmA2VTL_I0IdJNEgPFNw,8374
|
|
9
9
|
dcnum/feat/feat_background/bg_copy.py,sha256=EbeIy28gyPJr01Xens881IC1BtaTS5q-BkXPd3b6cLk,726
|
|
10
|
-
dcnum/feat/feat_background/bg_roll_median.py,sha256=
|
|
11
|
-
dcnum/feat/feat_background/bg_sparse_median.py,sha256=
|
|
10
|
+
dcnum/feat/feat_background/bg_roll_median.py,sha256=HgiGoyfLkygIlCoo8cBbf3gQt5uvM2S6_ez_V1hhCb4,12834
|
|
11
|
+
dcnum/feat/feat_background/bg_sparse_median.py,sha256=LbWbDxAruGagidHt9wybyqkXp9OKi3eWXceujirpsqY,17608
|
|
12
12
|
dcnum/feat/feat_brightness/__init__.py,sha256=o6AebVlmydwNgVF5kW6ITqJyFreoKrU3Ki_3EC8If-s,155
|
|
13
13
|
dcnum/feat/feat_brightness/bright_all.py,sha256=Z5b-xkw7g7ejMpbGmdUqrxGRymqFhAQsZ938gaGXk9Y,3102
|
|
14
14
|
dcnum/feat/feat_brightness/common.py,sha256=JX49EszYDmnvoOKXFVV1CalEIWRmOuY5EryNbqGbdac,156
|
|
@@ -19,7 +19,7 @@ dcnum/feat/feat_texture/__init__.py,sha256=6StM9S540UVtdFFR3bHa7nfCTomeVdoo7Uy9C
|
|
|
19
19
|
dcnum/feat/feat_texture/common.py,sha256=COXHpXS-7DMouGu3WF83I76L02Sr7P9re4lxajh6g0E,439
|
|
20
20
|
dcnum/feat/feat_texture/tex_all.py,sha256=eGjjNfPpfZw7FA_VNFCIMiU38KD0qcGbxLciYy-tCiA,4097
|
|
21
21
|
dcnum/logic/__init__.py,sha256=7J3GrwJInNQbrLk61HRIV7X7p69TAIbMYpR34hh6u14,177
|
|
22
|
-
dcnum/logic/ctrl.py,sha256=
|
|
22
|
+
dcnum/logic/ctrl.py,sha256=Gd17hdEYTxvdXZkTqqul4WUCu5wf4Gf0f8LZJ9EX83k,26725
|
|
23
23
|
dcnum/logic/job.py,sha256=M0Q-Rfcm-zkTXTQc79W6YSNUjUlgmRPG0Ikbdn1aOpY,4608
|
|
24
24
|
dcnum/logic/json_encoder.py,sha256=dy44ArmdnxpUfxxONmKdIv-fde3aTXPjZDN0HPATaxs,467
|
|
25
25
|
dcnum/meta/__init__.py,sha256=cQT_HN5yDKzMnZM8CUyNmeA68OhE3ENO_rvFmgDj95c,40
|
|
@@ -27,19 +27,19 @@ dcnum/meta/ppid.py,sha256=_xUqJal4wBqgic2aRN3ZMMteTggHeYGs44nrYbTKlpQ,8107
|
|
|
27
27
|
dcnum/read/__init__.py,sha256=iV2wrBMdwJgpXaphNiiAVybndDzTTv0CAGRNXyvxcLY,157
|
|
28
28
|
dcnum/read/cache.py,sha256=HXbRFyTNT08_imv2460hMKVrfRrU6WnbJoO71HR1j8E,5800
|
|
29
29
|
dcnum/read/const.py,sha256=SVlvEJiRIHyTyUlWG24_ogcnT5nTxCi0CRslNuNP56I,282
|
|
30
|
-
dcnum/read/hdf5_data.py,sha256=
|
|
30
|
+
dcnum/read/hdf5_data.py,sha256=tRSfBmrAZo7yNAnFLMj9yfh26_23pHxkZclX6RyzcYg,18864
|
|
31
31
|
dcnum/segm/__init__.py,sha256=iiq_1A9DU5wMUcKnsZ53E7NyzCkbZCJeUDimzunE-OM,247
|
|
32
32
|
dcnum/segm/segm_thresh.py,sha256=aLVTydPjbrgKDkZFY3Ew5CX-miwOw71meHfxcO5EjCc,1176
|
|
33
|
-
dcnum/segm/segmenter.py,sha256=
|
|
33
|
+
dcnum/segm/segmenter.py,sha256=g0sDBawObeMu4nhZ9rT8lqErsYYu3KApIKGW96aFvr8,10897
|
|
34
34
|
dcnum/segm/segmenter_cpu.py,sha256=tCY105rVr9_0RIq2618qnF1ueHRj7UtuK_nUBoAg-nY,10743
|
|
35
35
|
dcnum/segm/segmenter_gpu.py,sha256=tL2X5BN0jKmhC7wgfG0hygd-6UpG1ZCVuKe5OP1qde0,2133
|
|
36
36
|
dcnum/segm/segmenter_manager_thread.py,sha256=2znDaKedSueomcU1pbHtFmVcGoHzp--sf494VgJF_Tk,5342
|
|
37
37
|
dcnum/write/__init__.py,sha256=Cpn3LqL18hh8OScUnGp_AnNfpWPpKW-oAJZH6ot7aRA,241
|
|
38
|
-
dcnum/write/deque_writer_thread.py,sha256=
|
|
38
|
+
dcnum/write/deque_writer_thread.py,sha256=KpJ6po8JPlM696MITN-bhNnWQcy9E-qlhg9g-uzoPZg,1710
|
|
39
39
|
dcnum/write/queue_collector_thread.py,sha256=YQ6pvKNmCDf1C6HVx6gOA-q-FBoI6nkhOo-tAVYnyag,11906
|
|
40
|
-
dcnum/write/writer.py,sha256=
|
|
41
|
-
dcnum-0.16.
|
|
42
|
-
dcnum-0.16.
|
|
43
|
-
dcnum-0.16.
|
|
44
|
-
dcnum-0.16.
|
|
45
|
-
dcnum-0.16.
|
|
40
|
+
dcnum/write/writer.py,sha256=QGYNda102f2_12YWXu5WEBEQaTXhNnuQ20g-Dej-cek,10535
|
|
41
|
+
dcnum-0.16.7.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
|
|
42
|
+
dcnum-0.16.7.dist-info/METADATA,sha256=-FkRf9AUrKMuhTWwUp4SQnkYOKZ4KK4s6inH0Q2rZ4Q,2194
|
|
43
|
+
dcnum-0.16.7.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
|
44
|
+
dcnum-0.16.7.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
|
|
45
|
+
dcnum-0.16.7.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|