celldetective 1.3.5__py3-none-any.whl → 1.3.6.post2__py3-none-any.whl

celldetective/scripts/segment_cells.py

@@ -8,7 +8,7 @@ import os
 import json
 from stardist.models import StarDist2D
 from cellpose.models import CellposeModel
-from celldetective.io import locate_segmentation_model, auto_load_number_of_frames, load_frames
+from celldetective.io import locate_segmentation_model, auto_load_number_of_frames, load_frames, extract_position_name
 from celldetective.utils import interpolate_nan, _estimate_scale_factor, _extract_channel_indices_from_config, ConfigSectionMap, _extract_nbr_channels_from_config, _get_img_num_per_channel
 from pathlib import Path, PurePath
 from glob import glob
@@ -44,6 +44,7 @@ if use_gpu=='True' or use_gpu=='true' or use_gpu=='1':
 	n_threads = 1  # avoid misbehavior on GPU with multithreading
 else:
 	use_gpu = False
+	#n_threads = 1 # force 1 threads since all CPUs seem to be in use anyway
 
 if not use_gpu:
 	os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
@@ -60,20 +61,22 @@ parent1 = Path(pos).parent
 expfolder = parent1.parent
 config = PurePath(expfolder,Path("config.ini"))
 assert os.path.exists(config),'The configuration file for the experiment could not be located. Abort.'
+
+print(f"Position: {extract_position_name(pos)}...")
 print("Configuration file: ",config)
+print(f"Population: {mode}...")
 
 ####################################
 # Check model requirements #########
 ####################################
 
 modelpath = os.sep.join([os.path.split(os.path.dirname(os.path.realpath(__file__)))[0],"models"])
-print(modelpath)
 model_complete_path = locate_segmentation_model(modelname)
 if model_complete_path is None:
 	print('Model could not be found. Abort.')
 	os.abort()
 else:
-	print(f'Model successfully located in {model_complete_path}')
+	print(f'Model path: {model_complete_path}...')
 
 # load config
 assert os.path.exists(model_complete_path+"config_input.json"),'The configuration for the inputs to the model could not be located. Abort.'
@@ -86,7 +89,7 @@ required_channels = input_config["channels"]
 channel_indices = _extract_channel_indices_from_config(config, required_channels)
 print(f'Required channels: {required_channels} located at channel indices {channel_indices}.')
 required_spatial_calibration = input_config['spatial_calibration']
-print(f'Expected spatial calibration is {required_spatial_calibration}.')
+print(f'Spatial calibration expected by the model: {required_spatial_calibration}...')
 
 normalization_percentile = input_config['normalization_percentile']
 normalization_clip = input_config['normalization_clip']
@@ -117,18 +120,19 @@ if model_type=='cellpose':
 	flow_threshold = input_config['flow_threshold']
 
 scale = _estimate_scale_factor(spatial_calibration, required_spatial_calibration)
-print(f"Scale = {scale}...")
+print(f"Scale: {scale}...")
 
 nbr_channels = _extract_nbr_channels_from_config(config)
-print(f'Number of channels in the input movie: {nbr_channels}')
+#print(f'Number of channels in the input movie: {nbr_channels}')
 img_num_channels = _get_img_num_per_channel(channel_indices, int(len_movie), nbr_channels)
 
 # If everything OK, prepare output, load models
-print('Erasing previous segmentation folder.')
 if os.path.exists(pos+label_folder):
+	print('Erasing the previous labels folder...')
 	rmtree(pos+label_folder)
 os.mkdir(pos+label_folder)
-print(f'Folder {pos+label_folder} successfully generated.')
+print(f'Labels folder successfully generated...')
+
 log=f'segmentation model: {modelname}\n'
 with open(pos+f'log_{mode}.json', 'a') as f:
 	f.write(f'{datetime.datetime.now()} SEGMENT \n')
@@ -137,6 +141,7 @@ with open(pos+f'log_{mode}.json', 'a') as f:
 
 # Loop over all frames and segment
 def segment_index(indices):
+
 	global scale
 
 	if model_type=='stardist':
@@ -179,8 +184,7 @@ def segment_index(indices):
 		f = np.moveaxis([interpolate_nan(f[:,:,c].copy()) for c in range(f.shape[-1])],0,-1)
 
 		if np.any(img_num_channels[:,t]==-1):
-			f[:,:,np.where(img_num_channels[:,t]==-1)[0]] = 0.
-		
+			f[:,:,np.where(img_num_channels[:,t]==-1)[0]] = 0.	
 
 		if model_type=="stardist":
 			Y_pred, details = model.predict_instances(f, n_tiles=model._guess_n_tiles(f), show_tile_progress=False, verbose=False)
@@ -206,6 +210,10 @@ def segment_index(indices):
 		del Y_pred;
 		gc.collect()
 
+	return
+
+
+print(f"Starting the segmentation with {n_threads} thread(s) and GPU={use_gpu}...")
 
 import concurrent.futures
 
@@ -214,16 +222,12 @@ indices = list(range(img_num_channels.shape[1]))
 chunks = np.array_split(indices, n_threads)
 
 with concurrent.futures.ThreadPoolExecutor() as executor:
-	executor.map(segment_index, chunks)
-
-# threads = []
-# for i in range(n_threads):
-# 	thread_i = threading.Thread(target=segment_index, args=[chunks[i]])
-# 	threads.append(thread_i)
-# for th in threads:
-# 	th.start()
-# for th in threads:
-# 	th.join()
+	results = executor.map(segment_index, chunks)
+	try:
+		for i,return_value in enumerate(results):
+			print(f"Thread {i} output check: ",return_value)
+	except Exception as e:
+		print("Exception: ", e)
 
 print('Done.')
 

celldetective/scripts/segment_cells_thresholds.py

@@ -5,7 +5,7 @@ Copright © 2022 Laboratoire Adhesion et Inflammation, Authored by Remy Torro.
 import argparse
 import os
 import json
-from celldetective.io import auto_load_number_of_frames, load_frames
+from celldetective.io import auto_load_number_of_frames, load_frames, extract_position_name
 from celldetective.segmentation import segment_frame_from_thresholds
 from celldetective.utils import _extract_channel_indices_from_config, ConfigSectionMap, _extract_nbr_channels_from_config, _get_img_num_per_channel, extract_experiment_channels
 from pathlib import Path, PurePath
@@ -48,8 +48,6 @@ else:
 	print('The configuration path is not valid. Abort.')
 	os.abort()
 
-print('The following instructions were successfully loaded: ', threshold_instructions)
-
 if mode.lower()=="target" or mode.lower()=="targets":
 	label_folder = "labels_targets"
 elif mode.lower()=="effector" or mode.lower()=="effectors":
@@ -60,12 +58,14 @@ parent1 = Path(pos).parent
 expfolder = parent1.parent
 config = PurePath(expfolder,Path("config.ini"))
 assert os.path.exists(config),'The configuration file for the experiment could not be located. Abort.'
-print("Configuration file: ",config)
 
+print(f"Position: {extract_position_name(pos)}...")
+print("Configuration file: ",config)
+print(f"Population: {mode}...")
 
 channel_indices = _extract_channel_indices_from_config(config, required_channels)
 # need to abort if channel not found
-print(f'Required channels: {required_channels} located at channel indices {channel_indices}.')
+print(f'Required channels: {required_channels} located at channel indices {channel_indices}...')
 
 threshold_instructions.update({'target_channel': channel_indices[0]})
 
@@ -86,15 +86,15 @@ if len_movie_auto is not None:
 	len_movie = len_movie_auto
 
 nbr_channels = _extract_nbr_channels_from_config(config)
-print(f'Number of channels in the input movie: {nbr_channels}')
+#print(f'Number of channels in the input movie: {nbr_channels}')
 img_num_channels = _get_img_num_per_channel(np.arange(nbr_channels), len_movie, nbr_channels)
 
 # If everything OK, prepare output, load models
-print('Erasing previous segmentation folder.')
 if os.path.exists(os.sep.join([pos,label_folder])):
+	print('Erasing the previous labels folder...')
 	rmtree(os.sep.join([pos,label_folder]))
 os.mkdir(os.sep.join([pos,label_folder]))
-print(f'Folder {os.sep.join([pos,label_folder])} successfully generated.')
+print(f'Labels folder successfully generated...')
 
 if equalize:
 	f_reference = load_frames(img_num_channels[:,equalize_time], file, scale=None, normalize_input=False)
@@ -103,7 +103,7 @@ else:
 	f_reference = None
 
 threshold_instructions.update({'equalize_reference': f_reference})
-print(threshold_instructions)
+print(f"Instructions: {threshold_instructions}...")
 
 # Loop over all frames and segment
 def segment_index(indices):
@@ -119,6 +119,11 @@ def segment_index(indices):
 		del mask;
 		gc.collect()
 
+	return 
+
+
+print(f"Starting the segmentation with {n_threads} thread(s)...")
+
 import concurrent.futures
 
 # Multithreading
@@ -126,20 +131,15 @@ indices = list(range(img_num_channels.shape[1]))
 chunks = np.array_split(indices, n_threads)
 
 with concurrent.futures.ThreadPoolExecutor() as executor:
-	executor.map(segment_index, chunks)
-
-# indices = list(range(img_num_channels.shape[1]))
-# chunks = np.array_split(indices, n_threads)
-# threads = []
-# for i in range(n_threads):
-# 	thread_i = threading.Thread(target=segment_index, args=[chunks[i]])
-# 	threads.append(thread_i)
-# for th in threads:
-# 	th.start()
-# for th in threads:
-# 	th.join()
+	results = executor.map(segment_index, chunks)
+	try:
+		for i,return_value in enumerate(results):
+			print(f"Thread {i} output check: ",return_value)
+	except Exception as e:
+		print("Exception: ", e)
 
 print('Done.')
+
 gc.collect()
 
 

celldetective/scripts/track_cells.py

@@ -6,7 +6,7 @@ import argparse
 import datetime
 import os
 import json
-from celldetective.io import auto_load_number_of_frames, load_frames, interpret_tracking_configuration
+from celldetective.io import auto_load_number_of_frames, load_frames, interpret_tracking_configuration, extract_position_name
 from celldetective.utils import extract_experiment_channels, ConfigSectionMap, _get_img_num_per_channel, extract_experiment_channels
 from celldetective.measure import drop_tonal_features, measure_features
 from celldetective.tracking import track
@@ -59,6 +59,10 @@ expfolder = parent1.parent
 config = PurePath(expfolder,Path("config.ini"))
 assert os.path.exists(config),'The configuration file for the experiment could not be located. Abort.'
 
+print(f"Position: {extract_position_name(pos)}...")
+print("Configuration file: ",config)
+print(f"Population: {mode}...")
+
 # from exp config fetch spatial calib, channel names
 movie_prefix = ConfigSectionMap(config,"MovieSettings")["movie_prefix"]
 spatial_calibration = float(ConfigSectionMap(config,"MovieSettings")["pxtoum"])
@@ -71,9 +75,10 @@ channel_names, channel_indices = extract_experiment_channels(config)
 nbr_channels = len(channel_names)
 
 # from tracking instructions, fetch btrack config, features, haralick, clean_traj, idea: fetch custom timeline?
+print('Looking for tracking instruction file...')
 instr_path = PurePath(expfolder,Path(f"{instruction_file}"))
 if os.path.exists(instr_path):
-	print(f"Tracking instructions for the {mode} population have been successfully loaded...")
+	print(f"Tracking instruction file successfully loaded...")
 	with open(instr_path, 'r') as f:
 		instructions = json.load(f)
 	btrack_config = interpret_tracking_configuration(instructions['btrack_config_path'])
@@ -97,6 +102,16 @@ if os.path.exists(instr_path):
 		post_processing_options = instructions['post_processing_options']
 	else:
 		post_processing_options = None
+
+	btrack_option = True
+	if 'btrack_option' in instructions:
+		btrack_option = instructions['btrack_option']
+	search_range = None
+	if 'search_range' in instructions:
+		search_range = instructions['search_range']
+	memory = None
+	if 'memory' in instructions:
+		memory = instructions['memory']
 else:
 	print('Tracking instructions could not be located... Using a standard bTrack motion model instead...')
 	btrack_config = interpret_tracking_configuration(None)
@@ -104,7 +119,9 @@ else:
 	mask_channels = None
 	haralick_options = None
 	post_processing_options = None
-
+	btrack_option = True
+	memory = None
+	search_range = None
 if features is None:
 	features = []
 
@@ -147,7 +164,15 @@ with open(pos+f'log_{mode}.json', 'a') as f:
 	f.write(f'{datetime.datetime.now()} TRACK \n')
 	f.write(log+"\n")
 
+
+if not btrack_option:
+	features = []
+	channel_names = None
+	haralick_options = None
+
+
 def measure_index(indices):
+
 	for t in tqdm(indices,desc="frame"):
 		
 		# Load channels at time t
@@ -160,24 +185,27 @@ def measure_index(indices):
 		df_props.rename(columns={'centroid-1': 'x', 'centroid-0': 'y'},inplace=True)
 		df_props['t'] = int(t)
 		timestep_dataframes.append(df_props)
+	return 
+
+
+
+print(f"Measuring features with {n_threads} thread(s)...")
+
+import concurrent.futures
 
 # Multithreading
 indices = list(range(img_num_channels.shape[1]))
 chunks = np.array_split(indices, n_threads)
 
-import concurrent.futures
-
 with concurrent.futures.ThreadPoolExecutor() as executor:
-	executor.map(measure_index, chunks)
+	results = executor.map(measure_index, chunks)
+	try:
+		for i,return_value in enumerate(results):
+			print(f"Thread {i} output check: ",return_value)
+	except Exception as e:
+		print("Exception: ", e)
 
-# threads = []
-# for i in range(n_threads):
-# 	thread_i = threading.Thread(target=measure_index, args=[chunks[i]])
-# 	threads.append(thread_i)
-# for th in threads:
-# 	th.start()
-# for th in threads:
-# 	th.join()
+print('Features successfully measured...')
 
 df = pd.concat(timestep_dataframes)	
 df.reset_index(inplace=True, drop=True)
@@ -193,6 +221,13 @@ if mask_channels is not None:
 		df = df.drop(cols_to_drop, axis=1)
 
 # do tracking
+if btrack_option:
+	tracker = 'bTrack'
+else:
+	tracker = 'trackpy'
+
+print(f"Start the tracking step using the {tracker} tracker...")
+
 trajectories, napari_data = track(None,
 					configuration=btrack_config,
 					objects=df, 
@@ -203,15 +238,18 @@ trajectories, napari_data = track(None,
 		  			track_kwargs={'step_size': 100}, 
 		  			clean_trajectories_kwargs=post_processing_options, 
 		  			volume=(shape_x, shape_y),
+		  			btrack_option=btrack_option, 
+		  			search_range=search_range,
+		  			memory=memory,
 		  			)
+print(f"Tracking successfully performed...")
 
 # out trajectory table, create POSITION_X_um, POSITION_Y_um, TIME_min (new ones)
-# Save napari data
+# Save napari data # deprecated, should disappear progressively
 np.save(pos+os.sep.join(['output', 'tables', napari_name]), napari_data, allow_pickle=True)
-print(f"napari data successfully saved in {pos+os.sep.join(['output', 'tables'])}")
 
 trajectories.to_csv(pos+os.sep.join(['output', 'tables', table_name]), index=False)
-print(f"Table {table_name} successfully saved in {os.sep.join(['output', 'tables'])}")
+print(f"Trajectory table successfully exported in {os.sep.join(['output', 'tables'])}...")
 
 if os.path.exists(pos+os.sep.join(['output', 'tables', table_name.replace('.csv','.pkl')])):
 	os.remove(pos+os.sep.join(['output', 'tables', table_name.replace('.csv','.pkl')]))

celldetective/tracking.py

@@ -12,13 +12,14 @@ from celldetective.io import interpret_tracking_configuration
 
 import os
 import subprocess
+import trackpy as tp
 
 abs_path = os.sep.join([os.path.split(os.path.dirname(os.path.realpath(__file__)))[0],'celldetective'])
 
 def track(labels, configuration=None, stack=None, spatial_calibration=1, features=None, channel_names=None,
 		  haralick_options=None, return_napari_data=False, view_on_napari=False, mask_timepoints=None, mask_channels=None, volume=(2048,2048),
 		  optimizer_options = {'tm_lim': int(12e4)}, track_kwargs={'step_size': 100}, objects=None,
-		  clean_trajectories_kwargs=None, column_labels={'track': "TRACK_ID", 'time': 'FRAME', 'x': 'POSITION_X', 'y': 'POSITION_Y'},
+		  clean_trajectories_kwargs=None, btrack_option=True, search_range=None, memory=None,column_labels={'track': "TRACK_ID", 'time': 'FRAME', 'x': 'POSITION_X', 'y': 'POSITION_Y'},
 		  ):
 
 	"""
@@ -90,6 +91,12 @@ def track(labels, configuration=None, stack=None, spatial_calibration=1, feature
 	configuration = interpret_tracking_configuration(configuration)
 
 	if objects is None:
+		
+		if not btrack_option:
+			features = []
+			channel_names = None
+			haralick_options = None
+
 		objects = extract_objects_and_features(labels, stack, features, 
 										   channel_names=channel_names,
 										   haralick_options=haralick_options,
@@ -97,63 +104,81 @@ def track(labels, configuration=None, stack=None, spatial_calibration=1, feature
 										   mask_channels=mask_channels,
 										   )
 
-	columns = list(objects.columns)
-	to_remove = ['x','y','class_id','t']
-	for tr in to_remove:
-		try:
-			columns.remove(tr)
-		except:
-			print(f'column {tr} could not be found...')
-
-	scaler = StandardScaler()
-	if columns:
-		x = objects[columns].values
-		x_scaled = scaler.fit_transform(x)
-		df_temp = pd.DataFrame(x_scaled, columns=columns, index = objects.index)
-		objects[columns] = df_temp
-	else:
-		print('Warning: no features were passed to bTrack...')
+	if btrack_option:
+		columns = list(objects.columns)
+		to_remove = ['x','y','class_id','t']
+		for tr in to_remove:
+			try:
+				columns.remove(tr)
+			except:
+				print(f'column {tr} could not be found...')
 
-	# 2) track the objects
-	new_btrack_objects = localizations_to_objects(objects)
+		scaler = StandardScaler()
+		if columns:
+			x = objects[columns].values
+			x_scaled = scaler.fit_transform(x)
+			df_temp = pd.DataFrame(x_scaled, columns=columns, index = objects.index)
+			objects[columns] = df_temp
+		else:
+			print('Warning: no features were passed to bTrack...')
 
-	with BayesianTracker() as tracker:
+		# 2) track the objects
+		new_btrack_objects = localizations_to_objects(objects)
 
-		tracker.configure(configuration)
+		with BayesianTracker() as tracker:
 
-		if columns:
-			tracking_updates = ["motion","visual"]
-			#tracker.tracking_updates = ["motion","visual"]
-			tracker.features = columns
+			tracker.configure(configuration)
+
+			if columns:
+				tracking_updates = ["motion","visual"]
+				#tracker.tracking_updates = ["motion","visual"]
+				tracker.features = columns
+			else:
+				tracking_updates = ["motion"]
+			
+			tracker.append(new_btrack_objects)
+			tracker.volume = ((0,volume[0]), (0,volume[1]), (-1e5, 1e5)) #(-1e5, 1e5)
+			#print(tracker.volume)
+			tracker.track(tracking_updates=tracking_updates, **track_kwargs)
+			tracker.optimize(options=optimizer_options)
+
+			data, properties, graph = tracker.to_napari() #ndim=2
+		# do the table post processing and napari options
+		if data.shape[1]==4:
+			df = pd.DataFrame(data, columns=[column_labels['track'],column_labels['time'],column_labels['y'],column_labels['x']])
+		elif data.shape[1]==5:
+			df = pd.DataFrame(data, columns=[column_labels['track'],column_labels['time'],"z",column_labels['y'],column_labels['x']])
+			df = df.drop(columns=['z'])	
+		df[column_labels['x']+'_um'] = df[column_labels['x']]*spatial_calibration
+		df[column_labels['y']+'_um'] = df[column_labels['y']]*spatial_calibration
+
+	else:
+		properties = None
+		graph = {}
+		print(f"{objects=} {objects.columns=}")
+		objects = objects.rename(columns={"t": "frame"})
+		if search_range is not None and memory is not None:
+			data = tp.link(objects, search_range, memory=memory,link_strategy='auto')
 		else:
-			tracking_updates = ["motion"]
-		
-		tracker.append(new_btrack_objects)
-		tracker.volume = ((0,volume[0]), (0,volume[1]), (-1e5, 1e5)) #(-1e5, 1e5)
-		#print(tracker.volume)
-		tracker.track(tracking_updates=tracking_updates, **track_kwargs)
-		tracker.optimize(options=optimizer_options)
-
-		data, properties, graph = tracker.to_napari() #ndim=2
-
-	# do the table post processing and napari options
-	if data.shape[1]==4:
-		df = pd.DataFrame(data, columns=[column_labels['track'],column_labels['time'],column_labels['y'],column_labels['x']])
-	elif data.shape[1]==5:
-		df = pd.DataFrame(data, columns=[column_labels['track'],column_labels['time'],"z",column_labels['y'],column_labels['x']])
-		df = df.drop(columns=['z'])	
-	df[column_labels['x']+'_um'] = df[column_labels['x']]*spatial_calibration
-	df[column_labels['y']+'_um'] = df[column_labels['y']]*spatial_calibration
-
-	df = df.merge(pd.DataFrame(properties),left_index=True, right_index=True)
-	if columns:
-		x = df[columns].values
-		x_scaled = scaler.inverse_transform(x)
-		df_temp = pd.DataFrame(x_scaled, columns=columns, index = df.index)
-		df[columns] = df_temp
-
-	# set dummy features to NaN
-	df.loc[df['dummy'],['class_id']+columns] = np.nan 
+			print('Please provide a valid search range and memory value...')
+			return None
+		data['particle'] = data['particle'] + 1 # force track id to start at 1
+		df = data.rename(columns={'frame': column_labels['time'], 'x': column_labels['x'], 'y': column_labels['y'], 'particle': column_labels['track']})
+		df['state'] = 5.0; df['generation'] = 0.0; df['root'] = 1.0; df['parent'] = 1.0; df['dummy'] = False; df['z'] = 0.0;
+		data = df[[column_labels['track'],column_labels['time'],"z",column_labels['y'],column_labels['x']]].to_numpy()
+		print(f"{df=}")
+
+	if btrack_option:
+		df = df.merge(pd.DataFrame(properties),left_index=True, right_index=True)
+		if columns:
+			x = df[columns].values
+			x_scaled = scaler.inverse_transform(x)
+			df_temp = pd.DataFrame(x_scaled, columns=columns, index = df.index)
+			df[columns] = df_temp
+
+		# set dummy features to NaN
+		df.loc[df['dummy'],['class_id']+columns] = np.nan
+
 	df = df.sort_values(by=[column_labels['track'],column_labels['time']])
 	df = velocity_per_track(df, window_size=3, mode='bi')
 

celldetective/utils.py

@@ -424,7 +424,7 @@ def estimate_unreliable_edge(activation_protocol=[['gauss',2],['std',4]]):
 	else:
 		edge=0
 		for fct in activation_protocol:
-			if isinstance(fct[1],(int,np.int_)):
+			if isinstance(fct[1],(int,np.int_)) and not fct[0]=='invert':
 				edge+=fct[1]
 		return edge
 
@@ -1250,7 +1250,10 @@ def ConfigSectionMap(path,section):
 	Config = configparser.ConfigParser()
 	Config.read(path)
 	dict1 = {}
-	options = Config.options(section)
+	try:
+		options = Config.options(section)
+	except:
+		return None
 	for option in options:
 		try:
 			dict1[option] = Config.get(section, option)

{celldetective-1.3.5.dist-info → celldetective-1.3.6.post2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: celldetective
-Version: 1.3.5
+Version: 1.3.6.post2
 Summary: description
 Home-page: http://github.com/remyeltorro/celldetective
 Author: Rémy Torro
@@ -42,6 +42,7 @@ Requires-Dist: pytest-qt
 Requires-Dist: h5py
 Requires-Dist: cliffs_delta
 Requires-Dist: requests
+Requires-Dist: trackpy
 
 # Celldetective