celldetective 1.3.7.post1__py3-none-any.whl → 1.3.7.post2__py3-none-any.whl

celldetective/gui/processes/segment_cells.py

@@ -0,0 +1,327 @@
+from multiprocessing import Process
+import time
+import datetime
+import os
+import json
+from celldetective.io import extract_position_name, locate_segmentation_model, auto_load_number_of_frames, load_frames, _check_label_dims, _load_frames_to_segment
+from celldetective.utils import _rescale_labels, _segment_image_with_stardist_model, _segment_image_with_cellpose_model, _prep_stardist_model, _prep_cellpose_model, _get_normalize_kwargs_from_config, extract_experiment_channels, _estimate_scale_factor, _extract_channel_indices_from_config, ConfigSectionMap, _extract_nbr_channels_from_config, _get_img_num_per_channel
+from pathlib import Path, PurePath
+from glob import glob
+from shutil import rmtree
+from tqdm import tqdm
+import numpy as np
+from csbdeep.io import save_tiff_imagej_compatible
+from celldetective.segmentation import segment_frame_from_thresholds
+import gc
+from art import tprint
+
+import concurrent.futures
+
+class BaseSegmentProcess(Process):
+
+	def __init__(self, queue=None, process_args=None, *args, **kwargs):
+		
+		super().__init__(*args, **kwargs)
+		
+		self.queue = queue
+
+		if process_args is not None:
+			for key, value in process_args.items():
+				setattr(self, key, value)
+
+		tprint("Segment")
+
+		# Experiment
+		self.locate_experiment_config()
+
+		print(f"Position: {extract_position_name(self.pos)}...")
+		print("Configuration file: ",self.config)
+		print(f"Population: {self.mode}...")
+
+		self.extract_experiment_parameters()
+		self.detect_movie_length()
+		self.write_folders()
+
+	def write_folders(self):
+
+		self.mode = self.mode.lower()
+
+		if self.mode=="target" or self.mode=="targets":
+			self.label_folder = "labels_targets"
+		elif self.mode=="effector" or self.mode=="effectors":
+			self.label_folder = "labels_effectors"
+
+		if os.path.exists(self.pos+self.label_folder):
+			print('Erasing the previous labels folder...')
+			rmtree(self.pos+self.label_folder)
+		os.mkdir(self.pos+self.label_folder)
+		print(f'Labels folder successfully generated...')
+
+
+	def extract_experiment_parameters(self):
+
+		self.spatial_calibration = float(ConfigSectionMap(self.config,"MovieSettings")["pxtoum"])
+		self.len_movie = float(ConfigSectionMap(self.config,"MovieSettings")["len_movie"])
+		self.movie_prefix = ConfigSectionMap(self.config,"MovieSettings")["movie_prefix"]
+		self.nbr_channels = _extract_nbr_channels_from_config(self.config)
+		self.channel_names, self.channel_indices = extract_experiment_channels(self.config)
+
+	def locate_experiment_config(self):
+
+		parent1 = Path(self.pos).parent
+		expfolder = parent1.parent
+		self.config = PurePath(expfolder,Path("config.ini"))
+
+		if not os.path.exists(self.config):
+			print('The configuration file for the experiment could not be located. Abort.')
+			self.abort_process()
+
+	def detect_movie_length(self):
+
+		try:
+			self.file = glob(self.pos+f"movie/{self.movie_prefix}*.tif")[0]
+		except Exception as e:
+			print(f'Error {e}.\nMovie could not be found. Check the prefix.')
+			self.abort_process()
+
+		len_movie_auto = auto_load_number_of_frames(self.file)
+		if len_movie_auto is not None:
+			self.len_movie = len_movie_auto
+
+	def end_process(self):
+
+		self.terminate()
+		self.queue.put("finished")
+
+	def abort_process(self):
+		
+		self.terminate()
+		self.queue.put("error")
+
+
+class SegmentCellDLProcess(BaseSegmentProcess):
+	
+	def __init__(self, *args, **kwargs):
+		
+		super().__init__(*args, **kwargs)
+
+		self.check_gpu()
+
+		# Model
+		self.locate_model_path()
+		self.extract_model_input_parameters()
+		self.detect_channels()
+		self.detect_rescaling()
+
+		self.write_log()
+
+		self.sum_done = 0
+		self.t0 = time.time()
+
+	def extract_model_input_parameters(self):
+
+		self.required_channels = self.input_config["channels"]
+		self.normalize_kwargs = _get_normalize_kwargs_from_config(self.input_config)
+
+		self.model_type = self.input_config['model_type']
+		self.required_spatial_calibration = self.input_config['spatial_calibration']
+		print(f'Spatial calibration expected by the model: {self.required_spatial_calibration}...')
+		
+		if self.model_type=='cellpose':
+			self.diameter = self.input_config['diameter']
+			self.cellprob_threshold = self.input_config['cellprob_threshold']
+			self.flow_threshold = self.input_config['flow_threshold']
+
+	def write_log(self):
+
+		log=f'segmentation model: {self.model_name}\n'
+		with open(self.pos+f'log_{self.mode}.txt', 'a') as f:
+			f.write(f'{datetime.datetime.now()} SEGMENT \n')
+			f.write(log)
+
+	def detect_channels(self):
+		
+		self.channel_indices = _extract_channel_indices_from_config(self.config, self.required_channels)
+		print(f'Required channels: {self.required_channels} located at channel indices {self.channel_indices}.')
+		self.img_num_channels = _get_img_num_per_channel(self.channel_indices, int(self.len_movie), self.nbr_channels)
+
+	def detect_rescaling(self):
+
+		self.scale = _estimate_scale_factor(self.spatial_calibration, self.required_spatial_calibration)
+		print(f"Scale: {self.scale}...")
+
+	def locate_model_path(self):
+
+		self.model_complete_path = locate_segmentation_model(self.model_name)
+		if self.model_complete_path is None:
+			print('Model could not be found. Abort.')
+			self.abort_process()
+		else:
+			print(f'Model path: {self.model_complete_path}...')
+		
+		if not os.path.exists(self.model_complete_path+"config_input.json"):
+			print('The configuration for the inputs to the model could not be located. Abort.')
+			self.abort_process()
+
+		with open(self.model_complete_path+"config_input.json") as config_file:
+			self.input_config = json.load(config_file)
+					
+	def check_gpu(self):
+
+		if not self.use_gpu:
+			os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
+
+	def run(self):
+
+		try:
+
+			if self.model_type=='stardist':
+				model, scale_model = _prep_stardist_model(self.model_name, Path(self.model_complete_path).parent, use_gpu=self.use_gpu, scale=self.scale)
+
+			elif self.model_type=='cellpose':
+				model, scale_model = _prep_cellpose_model(self.model_name, self.model_complete_path, use_gpu=self.use_gpu, n_channels=len(self.required_channels), scale=self.scale)
+
+			for t in tqdm(range(self.len_movie),desc="frame"):
+				
+				f = _load_frames_to_segment(self.file, self.img_num_channels[:,t], scale_model=scale_model, normalize_kwargs=self.normalize_kwargs)
+
+				if self.model_type=="stardist":
+					Y_pred = _segment_image_with_stardist_model(f, model=model, return_details=False)
+
+				elif self.model_type=="cellpose":
+					Y_pred = _segment_image_with_cellpose_model(f, model=model, diameter=self.diameter, cellprob_threshold=self.cellprob_threshold, flow_threshold=self.flow_threshold)
+
+				if self.scale is not None:
+					Y_pred = _rescale_labels(Y_pred, scale_model=scale_model)
+				
+				Y_pred = _check_label_dims(Y_pred, file=self.file)
+
+				save_tiff_imagej_compatible(self.pos+os.sep.join([self.label_folder,f"{str(t).zfill(4)}.tif"]), Y_pred, axes='YX')
+				
+				del f;
+				del Y_pred;
+				gc.collect()
+				
+				# Send signal for progress bar
+				self.sum_done+=1/self.len_movie*100
+				mean_exec_per_step = (time.time() - self.t0) / (t+1)
+				pred_time = (self.len_movie - (t+1)) * mean_exec_per_step
+				self.queue.put([self.sum_done, pred_time])
+
+		except Exception as e:
+			print(e)
+
+		try:
+			del model
+		except:
+			pass		
+		
+		gc.collect()
+
+		# Send end signal
+		self.queue.put("finished")
+		self.queue.close()
+
+
+class SegmentCellThresholdProcess(BaseSegmentProcess):
+	
+	def __init__(self, *args, **kwargs):
+		
+		super().__init__(*args, **kwargs)
+
+		self.equalize = False
+
+		# Model
+		self.load_threshold_config()
+		self.extract_threshold_parameters()
+		self.detect_channels()
+		self.prepare_equalize()
+
+		self.write_log()
+
+		self.sum_done = 0
+		self.t0 = time.time()
+
+	def prepare_equalize(self):
+
+		if self.equalize:
+			f_reference = load_frames(self.img_num_channels[:,self.equalize_time], self.file, scale=None, normalize_input=False)
+			f_reference = f_reference[:,:,self.threshold_instructions['target_channel']]
+		else:
+			f_reference = None
+
+		self.threshold_instructions.update({'equalize_reference': f_reference})
+
+	def load_threshold_config(self):
+		
+		if os.path.exists(self.threshold_instructions):
+			with open(self.threshold_instructions, 'r') as f:
+				self.threshold_instructions = json.load(f)
+		else:
+			print('The configuration path is not valid. Abort.')
+			self.abort_process()
+
+	def extract_threshold_parameters(self):
+		
+		self.required_channels = [self.threshold_instructions['target_channel']]
+		if 'equalize_reference' in self.threshold_instructions:
+			self.equalize, self.equalize_time = self.threshold_instructions['equalize_reference']		
+
+	def write_log(self):
+
+		log=f'Threshold segmentation: {self.threshold_instructions}\n'
+		with open(self.pos+f'log_{self.mode}.txt', 'a') as f:
+			f.write(f'{datetime.datetime.now()} SEGMENT \n')
+			f.write(log)
+
+	def detect_channels(self):
+
+		self.channel_indices = _extract_channel_indices_from_config(self.config, self.required_channels)
+		print(f'Required channels: {self.required_channels} located at channel indices {self.channel_indices}.')
+		
+		self.img_num_channels = _get_img_num_per_channel(np.arange(self.nbr_channels), self.len_movie, self.nbr_channels)
+		self.threshold_instructions.update({'target_channel': self.channel_indices[0]})
+		self.threshold_instructions.update({'channel_names': self.channel_names})
+
+	def parallel_job(self, indices):
+
+		try:
+
+			for t in tqdm(indices,desc="frame"): #for t in tqdm(range(self.len_movie),desc="frame"):
+				
+				# Load channels at time t
+				f = load_frames(self.img_num_channels[:,t], self.file, scale=None, normalize_input=False)
+				mask = segment_frame_from_thresholds(f, **self.threshold_instructions)
+				save_tiff_imagej_compatible(os.sep.join([self.pos, self.label_folder, f"{str(t).zfill(4)}.tif"]), mask.astype(np.uint16), axes='YX')
+
+				del f;
+				del mask;
+				gc.collect()
+
+				# Send signal for progress bar
+				self.sum_done+=1/self.len_movie*100
+				mean_exec_per_step = (time.time() - self.t0) / (self.sum_done*self.len_movie / 100 + 1)
+				pred_time = (self.len_movie - (self.sum_done*self.len_movie / 100 + 1)) * mean_exec_per_step
+				self.queue.put([self.sum_done, pred_time])
+
+		except Exception as e:
+			print(e)
+
+		return
+
+	def run(self):
+
+		self.indices = list(range(self.img_num_channels.shape[1]))
+		chunks = np.array_split(self.indices, self.n_threads)
+
+		with concurrent.futures.ThreadPoolExecutor(max_workers=self.n_threads) as executor:
+			results = results = executor.map(self.parallel_job, chunks) #list(map(lambda x: executor.submit(self.parallel_job, x), chunks))
+			try:
+				for i,return_value in enumerate(results):
+					print(f"Thread {i} output check: ",return_value)
+			except Exception as e:
+				print("Exception: ", e)
+
+		# Send end signal
+		self.queue.put("finished")
+		self.queue.close()

celldetective/gui/processes/track_cells.py

@@ -0,0 +1,298 @@
+from multiprocessing import Process
+import time
+import datetime
+import os
+import json
+from celldetective.io import auto_load_number_of_frames, _load_frames_to_measure, locate_labels
+from celldetective.utils import extract_experiment_channels, ConfigSectionMap, _get_img_num_per_channel, _mask_intensity_measurements
+from pathlib import Path, PurePath
+from glob import glob
+from tqdm import tqdm
+import numpy as np
+import gc
+import concurrent.futures
+import datetime
+import os
+import json
+from celldetective.io import interpret_tracking_configuration
+from celldetective.utils import extract_experiment_channels
+from celldetective.measure import drop_tonal_features, measure_features
+from celldetective.tracking import track
+import pandas as pd
+from natsort import natsorted
+from art import tprint
+
+
+class TrackingProcess(Process):
+
+	def __init__(self, queue=None, process_args=None, *args, **kwargs):
+		
+		super().__init__(*args, **kwargs)
+		
+		self.queue = queue
+
+		if process_args is not None:
+			for key, value in process_args.items():
+				setattr(self, key, value)
+
+
+		tprint("Track")
+		self.timestep_dataframes = []
+
+		# Experiment		
+		self.prepare_folders()
+
+		self.locate_experiment_config()
+		self.extract_experiment_parameters()
+		self.read_tracking_instructions()
+		self.detect_movie_and_labels()
+		self.detect_channels()
+
+		self.write_log()
+
+		if not self.btrack_option:
+			self.features = []
+			self.channel_names = None
+			self.haralick_options = None
+
+		self.sum_done = 0
+		self.t0 = time.time()
+
+	def read_tracking_instructions(self):
+
+		instr_path = PurePath(self.expfolder,Path(f"{self.instruction_file}"))
+		if os.path.exists(instr_path):
+			print(f"Tracking instructions for the {self.mode} population have been successfully loaded...")
+			with open(instr_path, 'r') as f:
+				self.instructions = json.load(f)
+			
+			self.btrack_config = interpret_tracking_configuration(self.instructions['btrack_config_path'])
+
+			if 'features' in self.instructions:
+				self.features = self.instructions['features']
+			else:
+				self.features = None
+			
+			if 'mask_channels' in self.instructions:
+				self.mask_channels = self.instructions['mask_channels']
+			else:
+				self.mask_channels = None
+			
+			if 'haralick_options' in self.instructions:
+				self.haralick_options = self.instructions['haralick_options']
+			else:
+				self.haralick_options = None
+
+			if 'post_processing_options' in self.instructions:
+				self.post_processing_options = self.instructions['post_processing_options']
+			else:
+				self.post_processing_options = None
+			
+			self.btrack_option = True
+			if 'btrack_option' in self.instructions:
+				self.btrack_option = self.instructions['btrack_option']
+			self.search_range = None
+			if 'search_range' in self.instructions:
+				self.search_range = self.instructions['search_range']
+			self.memory = None
+			if 'memory' in self.instructions:
+				self.memory = self.instructions['memory']
+		else:
+			print('Tracking instructions could not be located... Using a standard bTrack motion model instead...')
+			self.btrack_config = interpret_tracking_configuration(None)
+			self.features = None
+			self.mask_channels = None
+			self.haralick_options = None
+			self.post_processing_options = None
+			self.btrack_option = True
+			self.memory = None
+			self.search_range = None
+
+		if self.features is None:
+			self.features = []
+
+	def detect_channels(self):
+		self.img_num_channels = _get_img_num_per_channel(self.channel_indices, self.len_movie, self.nbr_channels)
+
+	def write_log(self):
+
+		features_log=f'features: {self.features}'
+		mask_channels_log=f'mask_channels: {self.mask_channels}'
+		haralick_option_log=f'haralick_options: {self.haralick_options}'
+		post_processing_option_log=f'post_processing_options: {self.post_processing_options}'
+		log_list=[features_log, mask_channels_log, haralick_option_log, post_processing_option_log]
+		log='\n'.join(log_list)
+
+		with open(self.pos+f'log_{self.mode}.txt', 'a') as f:
+			f.write(f'{datetime.datetime.now()} TRACK \n')
+			f.write(log+"\n")
+
+	def prepare_folders(self):
+
+		if not os.path.exists(self.pos+"output"):
+			os.mkdir(self.pos+"output")
+
+		if not os.path.exists(self.pos+os.sep.join(["output","tables"])):
+			os.mkdir(self.pos+os.sep.join(["output","tables"]))
+
+		if self.mode.lower()=="target" or self.mode.lower()=="targets":
+			self.label_folder = "labels_targets"
+			self.instruction_file = os.sep.join(["configs", "tracking_instructions_targets.json"])
+			self.napari_name = "napari_target_trajectories.npy"
+			self.table_name = "trajectories_targets.csv"
+
+		elif self.mode.lower()=="effector" or self.mode.lower()=="effectors":
+			self.label_folder = "labels_effectors"
+			self.instruction_file = os.sep.join(["configs","tracking_instructions_effectors.json"])
+			self.napari_name = "napari_effector_trajectories.npy"
+			self.table_name = "trajectories_effectors.csv"
+
+	def extract_experiment_parameters(self):
+
+		self.movie_prefix = ConfigSectionMap(self.config,"MovieSettings")["movie_prefix"]
+		self.spatial_calibration = float(ConfigSectionMap(self.config,"MovieSettings")["pxtoum"])
+		self.time_calibration = float(ConfigSectionMap(self.config,"MovieSettings")["frametomin"])
+		self.len_movie = float(ConfigSectionMap(self.config,"MovieSettings")["len_movie"])
+		self.shape_x = int(ConfigSectionMap(self.config,"MovieSettings")["shape_x"])
+		self.shape_y = int(ConfigSectionMap(self.config,"MovieSettings")["shape_y"])
+
+		self.channel_names, self.channel_indices = extract_experiment_channels(self.config)
+		self.nbr_channels = len(self.channel_names)
+
+	def locate_experiment_config(self):
+
+		parent1 = Path(self.pos).parent
+		self.expfolder = parent1.parent
+		self.config = PurePath(self.expfolder,Path("config.ini"))
+
+		if not os.path.exists(self.config):
+			print('The configuration file for the experiment was not found...')
+			self.abort_process()
+
+	def detect_movie_and_labels(self):
+
+		self.label_path = natsorted(glob(self.pos+f"{self.label_folder}"+os.sep+"*.tif"))
+		if len(self.label_path)>0:
+			print(f"Found {len(self.label_path)} segmented frames...")
+		else:
+			print(f"No segmented frames have been found. Please run segmentation first. Abort...")
+			self.abort_process()
+
+		try:
+			self.file = glob(self.pos+f"movie/{self.movie_prefix}*.tif")[0]
+		except IndexError:
+			self.file = None
+			self.haralick_option = None
+			self.features = drop_tonal_features(self.features)
+			print('Movie could not be found. Check the prefix.')
+
+		len_movie_auto = auto_load_number_of_frames(self.file)
+		if len_movie_auto is not None:
+			self.len_movie = len_movie_auto
+
+	def parallel_job(self, indices):
+
+		props = []
+
+		try:
+
+			for t in tqdm(indices,desc="frame"):
+				
+				# Load channels at time t
+				img = _load_frames_to_measure(self.file, indices=self.img_num_channels[:,t])
+				lbl = locate_labels(self.pos, population=self.mode, frames=t)
+				if lbl is None:
+					continue
+
+				df_props = measure_features(img, lbl, features = self.features+['centroid'], border_dist=None, 
+												channels=self.channel_names, haralick_options=self.haralick_options, verbose=False)
+				df_props.rename(columns={'centroid-1': 'x', 'centroid-0': 'y'},inplace=True)
+				df_props['t'] = int(t)
+
+				props.append(df_props)
+
+				self.sum_done+=1/self.len_movie*50
+				mean_exec_per_step = (time.time() - self.t0) / (self.sum_done*self.len_movie / 50 + 1)
+				pred_time = (self.len_movie - (self.sum_done*self.len_movie / 50 + 1)) * mean_exec_per_step + 30
+				self.queue.put([self.sum_done, pred_time])
+				
+
+		except Exception as e:
+			print(e)
+
+		return props
+
+	def run(self):
+
+		self.indices = list(range(self.img_num_channels.shape[1]))
+		chunks = np.array_split(self.indices, self.n_threads)
+
+		self.timestep_dataframes = []
+		with concurrent.futures.ThreadPoolExecutor(max_workers=self.n_threads) as executor:
+			results = executor.map(self.parallel_job, chunks)
+			try:
+				for i,return_value in enumerate(results):
+					print(f'Thread {i} completed...')
+					#print(f"Thread {i} output check: ",return_value)
+					self.timestep_dataframes.extend(return_value)
+			except Exception as e:
+				print("Exception: ", e)
+
+		print('Features successfully measured...')
+
+		df = pd.concat(self.timestep_dataframes)	
+		df.reset_index(inplace=True, drop=True)
+		df = _mask_intensity_measurements(df, self.mask_channels)
+
+		# do tracking
+		if self.btrack_option:
+			tracker = 'bTrack'
+		else:
+			tracker = 'trackpy'
+
+		# do tracking
+		trajectories, napari_data = track(None,
+							configuration=self.btrack_config,
+							objects=df, 
+							spatial_calibration=self.spatial_calibration, 
+							channel_names=self.channel_names,
+							return_napari_data=True,
+				  			optimizer_options = {'tm_lim': int(12e4)}, 
+				  			track_kwargs={'step_size': 100}, 
+				  			clean_trajectories_kwargs=self.post_processing_options, 
+				  			volume=(self.shape_x, self.shape_y),
+				  			btrack_option=self.btrack_option, 
+				  			search_range=self.search_range,
+				  			memory=self.memory,
+				  			)
+		print(f"Tracking successfully performed...")
+
+		# out trajectory table, create POSITION_X_um, POSITION_Y_um, TIME_min (new ones)
+		# Save napari data
+		np.save(self.pos+os.sep.join(['output', 'tables', self.napari_name]), napari_data, allow_pickle=True)
+
+		trajectories.to_csv(self.pos+os.sep.join(['output', 'tables', self.table_name]), index=False)
+		print(f"Trajectory table successfully exported in {os.sep.join(['output', 'tables'])}...")
+
+		if os.path.exists(self.pos+os.sep.join(['output', 'tables', self.table_name.replace('.csv','.pkl')])):
+			os.remove(self.pos+os.sep.join(['output', 'tables', self.table_name.replace('.csv','.pkl')]))
+
+		del trajectories; del napari_data;
+		gc.collect()		
+		
+		# Send end signal
+		self.queue.put([100, 0])
+		time.sleep(1)
+		
+		self.queue.put("finished")
+		self.queue.close()
+
+	def end_process(self):
+
+		self.terminate()
+		self.queue.put("finished")
+
+	def abort_process(self):
+		
+		self.terminate()
+		self.queue.put("error")