celldetective 1.3.7.post1__py3-none-any.whl → 1.3.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- celldetective/_version.py +1 -1
- celldetective/gui/btrack_options.py +8 -8
- celldetective/gui/classifier_widget.py +8 -0
- celldetective/gui/configure_new_exp.py +1 -1
- celldetective/gui/json_readers.py +2 -4
- celldetective/gui/plot_signals_ui.py +38 -29
- celldetective/gui/process_block.py +1 -0
- celldetective/gui/processes/downloader.py +108 -0
- celldetective/gui/processes/measure_cells.py +346 -0
- celldetective/gui/processes/segment_cells.py +354 -0
- celldetective/gui/processes/track_cells.py +298 -0
- celldetective/gui/processes/train_segmentation_model.py +270 -0
- celldetective/gui/processes/train_signal_model.py +108 -0
- celldetective/gui/seg_model_loader.py +71 -25
- celldetective/gui/signal_annotator2.py +10 -7
- celldetective/gui/signal_annotator_options.py +1 -1
- celldetective/gui/tableUI.py +252 -20
- celldetective/gui/viewers.py +1 -1
- celldetective/io.py +53 -20
- celldetective/measure.py +12 -144
- celldetective/relative_measurements.py +40 -43
- celldetective/segmentation.py +48 -1
- celldetective/signals.py +84 -305
- celldetective/tracking.py +23 -24
- celldetective/utils.py +1 -1
- {celldetective-1.3.7.post1.dist-info → celldetective-1.3.8.dist-info}/METADATA +11 -2
- {celldetective-1.3.7.post1.dist-info → celldetective-1.3.8.dist-info}/RECORD +31 -25
- {celldetective-1.3.7.post1.dist-info → celldetective-1.3.8.dist-info}/WHEEL +1 -1
- {celldetective-1.3.7.post1.dist-info → celldetective-1.3.8.dist-info}/LICENSE +0 -0
- {celldetective-1.3.7.post1.dist-info → celldetective-1.3.8.dist-info}/entry_points.txt +0 -0
- {celldetective-1.3.7.post1.dist-info → celldetective-1.3.8.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
from multiprocessing import Process
|
|
2
|
+
import time
|
|
3
|
+
import datetime
|
|
4
|
+
import os
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path, PurePath
|
|
7
|
+
|
|
8
|
+
from celldetective.io import auto_load_number_of_frames, load_frames, locate_labels
|
|
9
|
+
from celldetective.utils import extract_experiment_channels, ConfigSectionMap, _get_img_num_per_channel
|
|
10
|
+
from celldetective.utils import remove_trajectory_measurements, _extract_coordinates_from_features, _remove_invalid_cols
|
|
11
|
+
from celldetective.measure import drop_tonal_features, measure_features, measure_isotropic_intensity, measure_radial_distance_to_center, center_of_mass_to_abs_coordinates
|
|
12
|
+
|
|
13
|
+
from glob import glob
|
|
14
|
+
from tqdm import tqdm
|
|
15
|
+
import numpy as np
|
|
16
|
+
import concurrent.futures
|
|
17
|
+
import pandas as pd
|
|
18
|
+
from natsort import natsorted
|
|
19
|
+
from art import tprint
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class MeasurementProcess(Process):
|
|
23
|
+
|
|
24
|
+
def __init__(self, queue=None, process_args=None):
|
|
25
|
+
|
|
26
|
+
super().__init__()
|
|
27
|
+
|
|
28
|
+
self.queue = queue
|
|
29
|
+
|
|
30
|
+
if process_args is not None:
|
|
31
|
+
for key, value in process_args.items():
|
|
32
|
+
setattr(self, key, value)
|
|
33
|
+
|
|
34
|
+
self.column_labels = {'track': "TRACK_ID", 'time': 'FRAME', 'x': 'POSITION_X', 'y': 'POSITION_Y'}
|
|
35
|
+
|
|
36
|
+
tprint("Measure")
|
|
37
|
+
|
|
38
|
+
# Experiment
|
|
39
|
+
self.prepare_folders()
|
|
40
|
+
|
|
41
|
+
self.locate_experiment_config()
|
|
42
|
+
self.extract_experiment_parameters()
|
|
43
|
+
self.read_measurement_instructions()
|
|
44
|
+
self.detect_movie_and_labels()
|
|
45
|
+
self.detect_tracks()
|
|
46
|
+
self.detect_channels()
|
|
47
|
+
|
|
48
|
+
self.check_possible_measurements()
|
|
49
|
+
|
|
50
|
+
self.write_log()
|
|
51
|
+
|
|
52
|
+
self.sum_done = 0
|
|
53
|
+
self.t0 = time.time()
|
|
54
|
+
|
|
55
|
+
def check_possible_measurements(self):
|
|
56
|
+
|
|
57
|
+
if (self.file is None) or (self.intensity_measurement_radii is None):
|
|
58
|
+
self.do_iso_intensities = False
|
|
59
|
+
print('Either no image, no positions or no radii were provided... Isotropic intensities will not be computed...')
|
|
60
|
+
else:
|
|
61
|
+
self.do_iso_intensities = True
|
|
62
|
+
|
|
63
|
+
if self.label_path is None:
|
|
64
|
+
self.do_features = False
|
|
65
|
+
print('No labels were provided... Features will not be computed...')
|
|
66
|
+
else:
|
|
67
|
+
self.do_features = True
|
|
68
|
+
|
|
69
|
+
if self.trajectories is None:
|
|
70
|
+
print('Use features as a substitute for the trajectory table.')
|
|
71
|
+
if 'label' not in self.features:
|
|
72
|
+
self.features.append('label')
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def read_measurement_instructions(self):
|
|
76
|
+
|
|
77
|
+
print('Looking for measurement instruction file...')
|
|
78
|
+
instr_path = PurePath(self.expfolder,Path(f"{self.instruction_file}"))
|
|
79
|
+
if os.path.exists(instr_path):
|
|
80
|
+
with open(instr_path, 'r') as f:
|
|
81
|
+
self.instructions = json.load(f)
|
|
82
|
+
print(f"Measurement instruction file successfully loaded...")
|
|
83
|
+
print(f"Instructions: {self.instructions}...")
|
|
84
|
+
|
|
85
|
+
if 'background_correction' in self.instructions:
|
|
86
|
+
self.background_correction = self.instructions['background_correction']
|
|
87
|
+
else:
|
|
88
|
+
self.background_correction = None
|
|
89
|
+
|
|
90
|
+
if 'features' in self.instructions:
|
|
91
|
+
self.features = self.instructions['features']
|
|
92
|
+
else:
|
|
93
|
+
self.features = None
|
|
94
|
+
|
|
95
|
+
if 'border_distances' in self.instructions:
|
|
96
|
+
self.border_distances = self.instructions['border_distances']
|
|
97
|
+
else:
|
|
98
|
+
self.border_distances = None
|
|
99
|
+
|
|
100
|
+
if 'spot_detection' in self.instructions:
|
|
101
|
+
self.spot_detection = self.instructions['spot_detection']
|
|
102
|
+
else:
|
|
103
|
+
self.spot_detection = None
|
|
104
|
+
|
|
105
|
+
if 'haralick_options' in self.instructions:
|
|
106
|
+
self.haralick_options = self.instructions['haralick_options']
|
|
107
|
+
else:
|
|
108
|
+
self.haralick_options = None
|
|
109
|
+
|
|
110
|
+
if 'intensity_measurement_radii' in self.instructions:
|
|
111
|
+
self.intensity_measurement_radii = self.instructions['intensity_measurement_radii']
|
|
112
|
+
else:
|
|
113
|
+
self.intensity_measurement_radii = None
|
|
114
|
+
|
|
115
|
+
if 'isotropic_operations' in self.instructions:
|
|
116
|
+
self.isotropic_operations = self.instructions['isotropic_operations']
|
|
117
|
+
else:
|
|
118
|
+
self.isotropic_operations = None
|
|
119
|
+
|
|
120
|
+
if 'clear_previous' in self.instructions:
|
|
121
|
+
self.clear_previous = self.instructions['clear_previous']
|
|
122
|
+
else:
|
|
123
|
+
self.clear_previous = True
|
|
124
|
+
|
|
125
|
+
else:
|
|
126
|
+
print('No measurement instructions found. Use default measurements.')
|
|
127
|
+
self.features = ['area', 'intensity_mean']
|
|
128
|
+
self.border_distances = None
|
|
129
|
+
self.haralick_options = None
|
|
130
|
+
self.clear_previous = False
|
|
131
|
+
self.background_correction = None
|
|
132
|
+
self.spot_detection = None
|
|
133
|
+
self.intensity_measurement_radii = 10
|
|
134
|
+
self.isotropic_operations = ['mean']
|
|
135
|
+
|
|
136
|
+
if self.features is None:
|
|
137
|
+
self.features = []
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def detect_channels(self):
|
|
141
|
+
self.img_num_channels = _get_img_num_per_channel(self.channel_indices, self.len_movie, self.nbr_channels)
|
|
142
|
+
|
|
143
|
+
def write_log(self):
|
|
144
|
+
|
|
145
|
+
features_log=f'features: {self.features}'
|
|
146
|
+
border_distances_log=f'border_distances: {self.border_distances}'
|
|
147
|
+
haralick_options_log=f'haralick_options: {self.haralick_options}'
|
|
148
|
+
background_correction_log=f'background_correction: {self.background_correction}'
|
|
149
|
+
spot_detection_log=f'spot_detection: {self.spot_detection}'
|
|
150
|
+
intensity_measurement_radii_log=f'intensity_measurement_radii: {self.intensity_measurement_radii}'
|
|
151
|
+
isotropic_options_log=f'isotropic_operations: {self.isotropic_operations} \n'
|
|
152
|
+
log='\n'.join([features_log,border_distances_log,haralick_options_log,background_correction_log,spot_detection_log,intensity_measurement_radii_log,isotropic_options_log])
|
|
153
|
+
with open(self.pos + f'log_{self.mode}.txt', 'a') as f:
|
|
154
|
+
f.write(f'{datetime.datetime.now()} MEASURE \n')
|
|
155
|
+
f.write(log+'\n')
|
|
156
|
+
|
|
157
|
+
def prepare_folders(self):
|
|
158
|
+
|
|
159
|
+
if self.mode.lower()=="target" or self.mode.lower()=="targets":
|
|
160
|
+
self.label_folder = "labels_targets"
|
|
161
|
+
self.table_name = "trajectories_targets.csv"
|
|
162
|
+
self.instruction_file = os.sep.join(["configs","measurement_instructions_targets.json"])
|
|
163
|
+
|
|
164
|
+
elif self.mode.lower()=="effector" or self.mode.lower()=="effectors":
|
|
165
|
+
self.label_folder = "labels_effectors"
|
|
166
|
+
self.table_name = "trajectories_effectors.csv"
|
|
167
|
+
self.instruction_file = os.sep.join(["configs","measurement_instructions_effectors.json"])
|
|
168
|
+
|
|
169
|
+
def extract_experiment_parameters(self):
|
|
170
|
+
|
|
171
|
+
self.movie_prefix = ConfigSectionMap(self.config,"MovieSettings")["movie_prefix"]
|
|
172
|
+
self.spatial_calibration = float(ConfigSectionMap(self.config,"MovieSettings")["pxtoum"])
|
|
173
|
+
self.time_calibration = float(ConfigSectionMap(self.config,"MovieSettings")["frametomin"])
|
|
174
|
+
self.len_movie = float(ConfigSectionMap(self.config,"MovieSettings")["len_movie"])
|
|
175
|
+
self.shape_x = int(ConfigSectionMap(self.config,"MovieSettings")["shape_x"])
|
|
176
|
+
self.shape_y = int(ConfigSectionMap(self.config,"MovieSettings")["shape_y"])
|
|
177
|
+
|
|
178
|
+
self.channel_names, self.channel_indices = extract_experiment_channels(self.config)
|
|
179
|
+
self.nbr_channels = len(self.channel_names)
|
|
180
|
+
|
|
181
|
+
def locate_experiment_config(self):
|
|
182
|
+
|
|
183
|
+
parent1 = Path(self.pos).parent
|
|
184
|
+
self.expfolder = parent1.parent
|
|
185
|
+
self.config = PurePath(self.expfolder,Path("config.ini"))
|
|
186
|
+
|
|
187
|
+
if not os.path.exists(self.config):
|
|
188
|
+
print('The configuration file for the experiment was not found...')
|
|
189
|
+
self.abort_process()
|
|
190
|
+
|
|
191
|
+
def detect_tracks(self):
|
|
192
|
+
|
|
193
|
+
# Load trajectories, add centroid if not in trajectory
|
|
194
|
+
self.trajectories = self.pos+os.sep.join(['output','tables', self.table_name])
|
|
195
|
+
if os.path.exists(self.trajectories):
|
|
196
|
+
print('trajectory exists...')
|
|
197
|
+
self.trajectories = pd.read_csv(self.trajectories)
|
|
198
|
+
if 'TRACK_ID' not in list(self.trajectories.columns):
|
|
199
|
+
self.do_iso_intensities = False
|
|
200
|
+
self.intensity_measurement_radii = None
|
|
201
|
+
if self.clear_previous:
|
|
202
|
+
print('No TRACK_ID... Clear previous measurements...')
|
|
203
|
+
self.trajectories = None #remove_trajectory_measurements(trajectories, column_labels)
|
|
204
|
+
self.do_features = True
|
|
205
|
+
self.features += ['centroid']
|
|
206
|
+
else:
|
|
207
|
+
if self.clear_previous:
|
|
208
|
+
print('TRACK_ID found... Clear previous measurements...')
|
|
209
|
+
self.trajectories = remove_trajectory_measurements(self.trajectories, self.column_labels)
|
|
210
|
+
else:
|
|
211
|
+
self.trajectories = None
|
|
212
|
+
self.do_features = True
|
|
213
|
+
self.features += ['centroid']
|
|
214
|
+
self.do_iso_intensities = False
|
|
215
|
+
|
|
216
|
+
def detect_movie_and_labels(self):
|
|
217
|
+
|
|
218
|
+
self.label_path = natsorted(glob(os.sep.join([self.pos, self.label_folder, '*.tif'])))
|
|
219
|
+
if len(self.label_path)>0:
|
|
220
|
+
print(f"Found {len(self.label_path)} segmented frames...")
|
|
221
|
+
else:
|
|
222
|
+
self.features = None
|
|
223
|
+
self.haralick_options = None
|
|
224
|
+
self.border_distances = None
|
|
225
|
+
self.label_path = None
|
|
226
|
+
|
|
227
|
+
try:
|
|
228
|
+
self.file = glob(self.pos+os.sep.join(["movie", f"{self.movie_prefix}*.tif"]))[0]
|
|
229
|
+
except IndexError:
|
|
230
|
+
self.file = None
|
|
231
|
+
self.haralick_option = None
|
|
232
|
+
self.features = drop_tonal_features(self.features)
|
|
233
|
+
|
|
234
|
+
len_movie_auto = auto_load_number_of_frames(self.file)
|
|
235
|
+
if len_movie_auto is not None:
|
|
236
|
+
self.len_movie = len_movie_auto
|
|
237
|
+
|
|
238
|
+
def parallel_job(self, indices):
|
|
239
|
+
|
|
240
|
+
measurements = []
|
|
241
|
+
|
|
242
|
+
for t in tqdm(indices,desc="frame"):
|
|
243
|
+
|
|
244
|
+
if self.file is not None:
|
|
245
|
+
img = load_frames(self.img_num_channels[:,t], self.file, scale=None, normalize_input=False)
|
|
246
|
+
|
|
247
|
+
if self.label_path is not None:
|
|
248
|
+
|
|
249
|
+
lbl = locate_labels(self.pos, population=self.mode, frames=t)
|
|
250
|
+
if lbl is None:
|
|
251
|
+
continue
|
|
252
|
+
|
|
253
|
+
if self.trajectories is not None:
|
|
254
|
+
|
|
255
|
+
positions_at_t = self.trajectories.loc[self.trajectories[self.column_labels['time']]==t].copy()
|
|
256
|
+
|
|
257
|
+
if self.do_features:
|
|
258
|
+
feature_table = measure_features(img, lbl, features=self.features, border_dist=self.border_distances,
|
|
259
|
+
channels=self.channel_names, haralick_options=self.haralick_options, verbose=False,
|
|
260
|
+
normalisation_list=self.background_correction, spot_detection=self.spot_detection)
|
|
261
|
+
if self.trajectories is None:
|
|
262
|
+
positions_at_t = _extract_coordinates_from_features(feature_table, timepoint=t)
|
|
263
|
+
column_labels = {'track': "ID", 'time': self.column_labels['time'], 'x': self.column_labels['x'],
|
|
264
|
+
'y': self.column_labels['y']}
|
|
265
|
+
feature_table.rename(columns={'centroid-1': 'POSITION_X', 'centroid-0': 'POSITION_Y'}, inplace=True)
|
|
266
|
+
|
|
267
|
+
if self.do_iso_intensities and not self.trajectories is None:
|
|
268
|
+
iso_table = measure_isotropic_intensity(positions_at_t, img, channels=self.channel_names, intensity_measurement_radii=self.intensity_measurement_radii, column_labels=self.column_labels, operations=self.isotropic_operations, verbose=False)
|
|
269
|
+
|
|
270
|
+
if self.do_iso_intensities and self.do_features and not self.trajectories is None:
|
|
271
|
+
measurements_at_t = iso_table.merge(feature_table, how='outer', on='class_id',suffixes=('_delme', ''))
|
|
272
|
+
measurements_at_t = measurements_at_t[[c for c in measurements_at_t.columns if not c.endswith('_delme')]]
|
|
273
|
+
elif self.do_iso_intensities * (not self.do_features) * (not self.trajectories is None):
|
|
274
|
+
measurements_at_t = iso_table
|
|
275
|
+
elif self.do_features:
|
|
276
|
+
measurements_at_t = positions_at_t.merge(feature_table, how='outer', on='class_id',suffixes=('_delme', ''))
|
|
277
|
+
measurements_at_t = measurements_at_t[[c for c in measurements_at_t.columns if not c.endswith('_delme')]]
|
|
278
|
+
|
|
279
|
+
measurements_at_t = center_of_mass_to_abs_coordinates(measurements_at_t)
|
|
280
|
+
measurements_at_t = measure_radial_distance_to_center(measurements_at_t, volume=img.shape, column_labels=self.column_labels)
|
|
281
|
+
|
|
282
|
+
self.sum_done+=1/self.len_movie*100
|
|
283
|
+
mean_exec_per_step = (time.time() - self.t0) / (self.sum_done*self.len_movie / 100 + 1)
|
|
284
|
+
pred_time = (self.len_movie - (self.sum_done*self.len_movie / 100 + 1)) * mean_exec_per_step
|
|
285
|
+
self.queue.put([self.sum_done, pred_time])
|
|
286
|
+
|
|
287
|
+
if measurements_at_t is not None:
|
|
288
|
+
measurements_at_t[self.column_labels['time']] = t
|
|
289
|
+
else:
|
|
290
|
+
measurements_at_t = pd.DataFrame()
|
|
291
|
+
|
|
292
|
+
measurements.append(measurements_at_t)
|
|
293
|
+
|
|
294
|
+
return measurements
|
|
295
|
+
|
|
296
|
+
def run(self):
|
|
297
|
+
|
|
298
|
+
self.indices = list(range(self.img_num_channels.shape[1]))
|
|
299
|
+
chunks = np.array_split(self.indices, self.n_threads)
|
|
300
|
+
|
|
301
|
+
self.timestep_dataframes = []
|
|
302
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=self.n_threads) as executor:
|
|
303
|
+
results = executor.map(self.parallel_job, chunks) #list(map(lambda x: executor.submit(self.parallel_job, x), chunks))
|
|
304
|
+
try:
|
|
305
|
+
for i,return_value in enumerate(results):
|
|
306
|
+
print(f'Thread {i} completed...')
|
|
307
|
+
self.timestep_dataframes.extend(return_value)
|
|
308
|
+
except Exception as e:
|
|
309
|
+
print("Exception: ", e)
|
|
310
|
+
|
|
311
|
+
print('Measurements successfully performed...')
|
|
312
|
+
|
|
313
|
+
if len(self.timestep_dataframes)>0:
|
|
314
|
+
|
|
315
|
+
df = pd.concat(self.timestep_dataframes)
|
|
316
|
+
|
|
317
|
+
if self.trajectories is not None:
|
|
318
|
+
df = df.sort_values(by=[self.column_labels['track'],self.column_labels['time']])
|
|
319
|
+
df = df.dropna(subset=[self.column_labels['track']])
|
|
320
|
+
else:
|
|
321
|
+
df['ID'] = np.arange(len(df))
|
|
322
|
+
df = df.sort_values(by=[self.column_labels['time'], 'ID'])
|
|
323
|
+
|
|
324
|
+
df = df.reset_index(drop=True)
|
|
325
|
+
df = _remove_invalid_cols(df)
|
|
326
|
+
|
|
327
|
+
df.to_csv(self.pos+os.sep.join(["output", "tables", self.table_name]), index=False)
|
|
328
|
+
print(f'Measurement table successfully exported in {os.sep.join(["output", "tables"])}...')
|
|
329
|
+
print('Done.')
|
|
330
|
+
else:
|
|
331
|
+
print('No measurement could be performed. Check your inputs.')
|
|
332
|
+
print('Done.')
|
|
333
|
+
|
|
334
|
+
# Send end signal
|
|
335
|
+
self.queue.put("finished")
|
|
336
|
+
self.queue.close()
|
|
337
|
+
|
|
338
|
+
def end_process(self):
|
|
339
|
+
|
|
340
|
+
self.terminate()
|
|
341
|
+
self.queue.put("finished")
|
|
342
|
+
|
|
343
|
+
def abort_process(self):
|
|
344
|
+
|
|
345
|
+
self.terminate()
|
|
346
|
+
self.queue.put("error")
|
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
from multiprocessing import Process
|
|
2
|
+
import time
|
|
3
|
+
import datetime
|
|
4
|
+
import os
|
|
5
|
+
import json
|
|
6
|
+
import numpy as np
|
|
7
|
+
from celldetective.io import extract_position_name, locate_segmentation_model, auto_load_number_of_frames, load_frames, _check_label_dims, _load_frames_to_segment
|
|
8
|
+
from celldetective.utils import _rescale_labels, _segment_image_with_stardist_model, _segment_image_with_cellpose_model, _prep_stardist_model, _prep_cellpose_model, _get_normalize_kwargs_from_config, extract_experiment_channels, _estimate_scale_factor, _extract_channel_indices_from_config, ConfigSectionMap, _extract_nbr_channels_from_config, _get_img_num_per_channel
|
|
9
|
+
|
|
10
|
+
from pathlib import Path, PurePath
|
|
11
|
+
from glob import glob
|
|
12
|
+
from shutil import rmtree
|
|
13
|
+
from tqdm import tqdm
|
|
14
|
+
import numpy as np
|
|
15
|
+
from csbdeep.io import save_tiff_imagej_compatible
|
|
16
|
+
from celldetective.segmentation import segment_frame_from_thresholds, merge_instance_segmentation
|
|
17
|
+
import gc
|
|
18
|
+
from art import tprint
|
|
19
|
+
|
|
20
|
+
import concurrent.futures
|
|
21
|
+
|
|
22
|
+
class BaseSegmentProcess(Process):
|
|
23
|
+
|
|
24
|
+
def __init__(self, queue=None, process_args=None, *args, **kwargs):
|
|
25
|
+
|
|
26
|
+
super().__init__(*args, **kwargs)
|
|
27
|
+
|
|
28
|
+
self.queue = queue
|
|
29
|
+
|
|
30
|
+
if process_args is not None:
|
|
31
|
+
for key, value in process_args.items():
|
|
32
|
+
setattr(self, key, value)
|
|
33
|
+
|
|
34
|
+
tprint("Segment")
|
|
35
|
+
|
|
36
|
+
# Experiment
|
|
37
|
+
self.locate_experiment_config()
|
|
38
|
+
|
|
39
|
+
print(f"Position: {extract_position_name(self.pos)}...")
|
|
40
|
+
print("Configuration file: ",self.config)
|
|
41
|
+
print(f"Population: {self.mode}...")
|
|
42
|
+
|
|
43
|
+
self.extract_experiment_parameters()
|
|
44
|
+
self.detect_movie_length()
|
|
45
|
+
self.write_folders()
|
|
46
|
+
|
|
47
|
+
def write_folders(self):
|
|
48
|
+
|
|
49
|
+
self.mode = self.mode.lower()
|
|
50
|
+
|
|
51
|
+
if self.mode=="target" or self.mode=="targets":
|
|
52
|
+
self.label_folder = "labels_targets"
|
|
53
|
+
elif self.mode=="effector" or self.mode=="effectors":
|
|
54
|
+
self.label_folder = "labels_effectors"
|
|
55
|
+
|
|
56
|
+
if os.path.exists(self.pos+self.label_folder):
|
|
57
|
+
print('Erasing the previous labels folder...')
|
|
58
|
+
rmtree(self.pos+self.label_folder)
|
|
59
|
+
os.mkdir(self.pos+self.label_folder)
|
|
60
|
+
print(f'Labels folder successfully generated...')
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def extract_experiment_parameters(self):
|
|
64
|
+
|
|
65
|
+
self.spatial_calibration = float(ConfigSectionMap(self.config,"MovieSettings")["pxtoum"])
|
|
66
|
+
self.len_movie = float(ConfigSectionMap(self.config,"MovieSettings")["len_movie"])
|
|
67
|
+
self.movie_prefix = ConfigSectionMap(self.config,"MovieSettings")["movie_prefix"]
|
|
68
|
+
self.nbr_channels = _extract_nbr_channels_from_config(self.config)
|
|
69
|
+
self.channel_names, self.channel_indices = extract_experiment_channels(self.config)
|
|
70
|
+
|
|
71
|
+
def locate_experiment_config(self):
|
|
72
|
+
|
|
73
|
+
parent1 = Path(self.pos).parent
|
|
74
|
+
expfolder = parent1.parent
|
|
75
|
+
self.config = PurePath(expfolder,Path("config.ini"))
|
|
76
|
+
|
|
77
|
+
if not os.path.exists(self.config):
|
|
78
|
+
print('The configuration file for the experiment could not be located. Abort.')
|
|
79
|
+
self.abort_process()
|
|
80
|
+
|
|
81
|
+
def detect_movie_length(self):
|
|
82
|
+
|
|
83
|
+
try:
|
|
84
|
+
self.file = glob(self.pos+f"movie/{self.movie_prefix}*.tif")[0]
|
|
85
|
+
except Exception as e:
|
|
86
|
+
print(f'Error {e}.\nMovie could not be found. Check the prefix.')
|
|
87
|
+
self.abort_process()
|
|
88
|
+
|
|
89
|
+
len_movie_auto = auto_load_number_of_frames(self.file)
|
|
90
|
+
if len_movie_auto is not None:
|
|
91
|
+
self.len_movie = len_movie_auto
|
|
92
|
+
|
|
93
|
+
def end_process(self):
|
|
94
|
+
|
|
95
|
+
self.terminate()
|
|
96
|
+
self.queue.put("finished")
|
|
97
|
+
|
|
98
|
+
def abort_process(self):
|
|
99
|
+
|
|
100
|
+
self.terminate()
|
|
101
|
+
self.queue.put("error")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class SegmentCellDLProcess(BaseSegmentProcess):
|
|
105
|
+
|
|
106
|
+
def __init__(self, *args, **kwargs):
|
|
107
|
+
|
|
108
|
+
super().__init__(*args, **kwargs)
|
|
109
|
+
|
|
110
|
+
self.check_gpu()
|
|
111
|
+
|
|
112
|
+
# Model
|
|
113
|
+
self.locate_model_path()
|
|
114
|
+
self.extract_model_input_parameters()
|
|
115
|
+
self.detect_channels()
|
|
116
|
+
self.detect_rescaling()
|
|
117
|
+
|
|
118
|
+
self.write_log()
|
|
119
|
+
|
|
120
|
+
self.sum_done = 0
|
|
121
|
+
self.t0 = time.time()
|
|
122
|
+
|
|
123
|
+
def extract_model_input_parameters(self):
|
|
124
|
+
|
|
125
|
+
self.required_channels = self.input_config["channels"]
|
|
126
|
+
self.normalize_kwargs = _get_normalize_kwargs_from_config(self.input_config)
|
|
127
|
+
|
|
128
|
+
self.model_type = self.input_config['model_type']
|
|
129
|
+
self.required_spatial_calibration = self.input_config['spatial_calibration']
|
|
130
|
+
print(f'Spatial calibration expected by the model: {self.required_spatial_calibration}...')
|
|
131
|
+
|
|
132
|
+
if self.model_type=='cellpose':
|
|
133
|
+
self.diameter = self.input_config['diameter']
|
|
134
|
+
self.cellprob_threshold = self.input_config['cellprob_threshold']
|
|
135
|
+
self.flow_threshold = self.input_config['flow_threshold']
|
|
136
|
+
|
|
137
|
+
def write_log(self):
|
|
138
|
+
|
|
139
|
+
log=f'segmentation model: {self.model_name}\n'
|
|
140
|
+
with open(self.pos+f'log_{self.mode}.txt', 'a') as f:
|
|
141
|
+
f.write(f'{datetime.datetime.now()} SEGMENT \n')
|
|
142
|
+
f.write(log)
|
|
143
|
+
|
|
144
|
+
def detect_channels(self):
|
|
145
|
+
|
|
146
|
+
self.channel_indices = _extract_channel_indices_from_config(self.config, self.required_channels)
|
|
147
|
+
print(f'Required channels: {self.required_channels} located at channel indices {self.channel_indices}.')
|
|
148
|
+
self.img_num_channels = _get_img_num_per_channel(self.channel_indices, int(self.len_movie), self.nbr_channels)
|
|
149
|
+
|
|
150
|
+
def detect_rescaling(self):
|
|
151
|
+
|
|
152
|
+
self.scale = _estimate_scale_factor(self.spatial_calibration, self.required_spatial_calibration)
|
|
153
|
+
print(f"Scale: {self.scale}...")
|
|
154
|
+
|
|
155
|
+
def locate_model_path(self):
|
|
156
|
+
|
|
157
|
+
self.model_complete_path = locate_segmentation_model(self.model_name)
|
|
158
|
+
if self.model_complete_path is None:
|
|
159
|
+
print('Model could not be found. Abort.')
|
|
160
|
+
self.abort_process()
|
|
161
|
+
else:
|
|
162
|
+
print(f'Model path: {self.model_complete_path}...')
|
|
163
|
+
|
|
164
|
+
if not os.path.exists(self.model_complete_path+"config_input.json"):
|
|
165
|
+
print('The configuration for the inputs to the model could not be located. Abort.')
|
|
166
|
+
self.abort_process()
|
|
167
|
+
|
|
168
|
+
with open(self.model_complete_path+"config_input.json") as config_file:
|
|
169
|
+
self.input_config = json.load(config_file)
|
|
170
|
+
|
|
171
|
+
def check_gpu(self):
|
|
172
|
+
|
|
173
|
+
if not self.use_gpu:
|
|
174
|
+
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
|
|
175
|
+
|
|
176
|
+
def run(self):
|
|
177
|
+
|
|
178
|
+
try:
|
|
179
|
+
|
|
180
|
+
if self.model_type=='stardist':
|
|
181
|
+
model, scale_model = _prep_stardist_model(self.model_name, Path(self.model_complete_path).parent, use_gpu=self.use_gpu, scale=self.scale)
|
|
182
|
+
|
|
183
|
+
elif self.model_type=='cellpose':
|
|
184
|
+
model, scale_model = _prep_cellpose_model(self.model_name, self.model_complete_path, use_gpu=self.use_gpu, n_channels=len(self.required_channels), scale=self.scale)
|
|
185
|
+
|
|
186
|
+
for t in tqdm(range(self.len_movie),desc="frame"):
|
|
187
|
+
|
|
188
|
+
f = _load_frames_to_segment(self.file, self.img_num_channels[:,t], scale_model=scale_model, normalize_kwargs=self.normalize_kwargs)
|
|
189
|
+
|
|
190
|
+
if self.model_type=="stardist":
|
|
191
|
+
Y_pred = _segment_image_with_stardist_model(f, model=model, return_details=False)
|
|
192
|
+
|
|
193
|
+
elif self.model_type=="cellpose":
|
|
194
|
+
Y_pred = _segment_image_with_cellpose_model(f, model=model, diameter=self.diameter, cellprob_threshold=self.cellprob_threshold, flow_threshold=self.flow_threshold)
|
|
195
|
+
|
|
196
|
+
if self.scale is not None:
|
|
197
|
+
Y_pred = _rescale_labels(Y_pred, scale_model=scale_model)
|
|
198
|
+
|
|
199
|
+
Y_pred = _check_label_dims(Y_pred, file=self.file)
|
|
200
|
+
|
|
201
|
+
save_tiff_imagej_compatible(self.pos+os.sep.join([self.label_folder,f"{str(t).zfill(4)}.tif"]), Y_pred, axes='YX')
|
|
202
|
+
|
|
203
|
+
del f;
|
|
204
|
+
del Y_pred;
|
|
205
|
+
gc.collect()
|
|
206
|
+
|
|
207
|
+
# Send signal for progress bar
|
|
208
|
+
self.sum_done+=1/self.len_movie*100
|
|
209
|
+
mean_exec_per_step = (time.time() - self.t0) / (t+1)
|
|
210
|
+
pred_time = (self.len_movie - (t+1)) * mean_exec_per_step
|
|
211
|
+
self.queue.put([self.sum_done, pred_time])
|
|
212
|
+
|
|
213
|
+
except Exception as e:
|
|
214
|
+
print(e)
|
|
215
|
+
|
|
216
|
+
try:
|
|
217
|
+
del model
|
|
218
|
+
except:
|
|
219
|
+
pass
|
|
220
|
+
|
|
221
|
+
gc.collect()
|
|
222
|
+
|
|
223
|
+
# Send end signal
|
|
224
|
+
self.queue.put("finished")
|
|
225
|
+
self.queue.close()
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
class SegmentCellThresholdProcess(BaseSegmentProcess):
|
|
229
|
+
|
|
230
|
+
def __init__(self, *args, **kwargs):
|
|
231
|
+
|
|
232
|
+
super().__init__(*args, **kwargs)
|
|
233
|
+
|
|
234
|
+
self.equalize = False
|
|
235
|
+
|
|
236
|
+
# Model
|
|
237
|
+
|
|
238
|
+
self.load_threshold_config()
|
|
239
|
+
self.extract_threshold_parameters()
|
|
240
|
+
self.detect_channels()
|
|
241
|
+
self.prepare_equalize()
|
|
242
|
+
|
|
243
|
+
self.write_log()
|
|
244
|
+
|
|
245
|
+
self.sum_done = 0
|
|
246
|
+
self.t0 = time.time()
|
|
247
|
+
|
|
248
|
+
def prepare_equalize(self):
|
|
249
|
+
|
|
250
|
+
for i in range(len(self.instructions)):
|
|
251
|
+
|
|
252
|
+
if self.equalize[i]:
|
|
253
|
+
f_reference = load_frames(self.img_num_channels[:,self.equalize_time[i]], self.file, scale=None, normalize_input=False)
|
|
254
|
+
f_reference = f_reference[:,:,self.instructions[i]['target_channel']]
|
|
255
|
+
else:
|
|
256
|
+
f_reference = None
|
|
257
|
+
|
|
258
|
+
self.instructions[i].update({'equalize_reference': f_reference})
|
|
259
|
+
|
|
260
|
+
def load_threshold_config(self):
|
|
261
|
+
|
|
262
|
+
self.instructions = []
|
|
263
|
+
for inst in self.threshold_instructions:
|
|
264
|
+
if os.path.exists(inst):
|
|
265
|
+
with open(inst, 'r') as f:
|
|
266
|
+
self.instructions.append(json.load(f))
|
|
267
|
+
else:
|
|
268
|
+
print('The configuration path is not valid. Abort.')
|
|
269
|
+
self.abort_process()
|
|
270
|
+
|
|
271
|
+
def extract_threshold_parameters(self):
|
|
272
|
+
|
|
273
|
+
self.required_channels = []
|
|
274
|
+
self.equalize = []
|
|
275
|
+
self.equalize_time = []
|
|
276
|
+
|
|
277
|
+
for i in range(len(self.instructions)):
|
|
278
|
+
ch = [self.instructions[i]['target_channel']]
|
|
279
|
+
self.required_channels.append(ch)
|
|
280
|
+
|
|
281
|
+
if 'equalize_reference' in self.instructions[i]:
|
|
282
|
+
equalize, equalize_time = self.instructions[i]['equalize_reference']
|
|
283
|
+
self.equalize.append(equalize)
|
|
284
|
+
self.equalize_time.append(equalize_time)
|
|
285
|
+
|
|
286
|
+
def write_log(self):
|
|
287
|
+
|
|
288
|
+
log=f'Threshold segmentation: {self.threshold_instructions}\n'
|
|
289
|
+
with open(self.pos+f'log_{self.mode}.txt', 'a') as f:
|
|
290
|
+
f.write(f'{datetime.datetime.now()} SEGMENT \n')
|
|
291
|
+
f.write(log)
|
|
292
|
+
|
|
293
|
+
def detect_channels(self):
|
|
294
|
+
|
|
295
|
+
for i in range(len(self.instructions)):
|
|
296
|
+
|
|
297
|
+
self.channel_indices = _extract_channel_indices_from_config(self.config, self.required_channels[i])
|
|
298
|
+
print(f'Required channels: {self.required_channels[i]} located at channel indices {self.channel_indices}.')
|
|
299
|
+
self.instructions[i].update({'target_channel': self.channel_indices[0]})
|
|
300
|
+
self.instructions[i].update({'channel_names': self.channel_names})
|
|
301
|
+
|
|
302
|
+
self.img_num_channels = _get_img_num_per_channel(np.arange(self.nbr_channels), self.len_movie, self.nbr_channels)
|
|
303
|
+
|
|
304
|
+
def parallel_job(self, indices):
|
|
305
|
+
|
|
306
|
+
try:
|
|
307
|
+
|
|
308
|
+
for t in tqdm(indices,desc="frame"): #for t in tqdm(range(self.len_movie),desc="frame"):
|
|
309
|
+
|
|
310
|
+
# Load channels at time t
|
|
311
|
+
masks = []
|
|
312
|
+
for i in range(len(self.instructions)):
|
|
313
|
+
f = load_frames(self.img_num_channels[:,t], self.file, scale=None, normalize_input=False)
|
|
314
|
+
mask = segment_frame_from_thresholds(f, **self.instructions[i])
|
|
315
|
+
#print(f'Frame {t}; segment with {self.instructions[i]=}...')
|
|
316
|
+
masks.append(mask)
|
|
317
|
+
|
|
318
|
+
if len(self.instructions)>1:
|
|
319
|
+
mask = merge_instance_segmentation(masks, mode='OR')
|
|
320
|
+
|
|
321
|
+
save_tiff_imagej_compatible(os.sep.join([self.pos, self.label_folder, f"{str(t).zfill(4)}.tif"]), mask.astype(np.uint16), axes='YX')
|
|
322
|
+
|
|
323
|
+
del f;
|
|
324
|
+
del mask;
|
|
325
|
+
gc.collect()
|
|
326
|
+
|
|
327
|
+
# Send signal for progress bar
|
|
328
|
+
self.sum_done+=1/self.len_movie*100
|
|
329
|
+
mean_exec_per_step = (time.time() - self.t0) / (self.sum_done*self.len_movie / 100 + 1)
|
|
330
|
+
pred_time = (self.len_movie - (self.sum_done*self.len_movie / 100 + 1)) * mean_exec_per_step
|
|
331
|
+
self.queue.put([self.sum_done, pred_time])
|
|
332
|
+
|
|
333
|
+
except Exception as e:
|
|
334
|
+
print(e)
|
|
335
|
+
|
|
336
|
+
return
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def run(self):
|
|
340
|
+
|
|
341
|
+
self.indices = list(range(self.img_num_channels.shape[1]))
|
|
342
|
+
chunks = np.array_split(self.indices, self.n_threads)
|
|
343
|
+
|
|
344
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=self.n_threads) as executor:
|
|
345
|
+
results = results = executor.map(self.parallel_job, chunks) #list(map(lambda x: executor.submit(self.parallel_job, x), chunks))
|
|
346
|
+
try:
|
|
347
|
+
for i,return_value in enumerate(results):
|
|
348
|
+
print(f"Thread {i} output check: ",return_value)
|
|
349
|
+
except Exception as e:
|
|
350
|
+
print("Exception: ", e)
|
|
351
|
+
|
|
352
|
+
# Send end signal
|
|
353
|
+
self.queue.put("finished")
|
|
354
|
+
self.queue.close()
|