celldetective 1.0.2__py3-none-any.whl

celldetective/scripts/track_cells.py

@@ -0,0 +1,206 @@
+"""
+Copright © 2022 Laboratoire Adhesion et Inflammation, Authored by Remy Torro.
+"""
+
+import argparse
+import os
+import json
+from celldetective.io import auto_load_number_of_frames, load_frames, interpret_tracking_configuration
+from celldetective.utils import extract_experiment_channels, _extract_channel_indices_from_config, _extract_channel_indices, ConfigSectionMap, _extract_nbr_channels_from_config, _get_img_num_per_channel, extract_experiment_channels
+from celldetective.measure import drop_tonal_features, measure_features
+from celldetective import track
+from pathlib import Path, PurePath
+from glob import glob
+from shutil import rmtree
+from tqdm import tqdm
+import numpy as np
+import pandas as pd
+import gc
+import os
+from natsort import natsorted
+from art import tprint
+from tifffile import imread
+import threading
+
+tprint("Track")
+
+parser = argparse.ArgumentParser(description="Segment a movie in position with the selected model",
+								formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-p',"--position", required=True, help="Path to the position")
+parser.add_argument("--mode", default="target", choices=["target","effector","targets","effectors"],help="Cell population of interest")
+parser.add_argument("--threads", default="1",help="Number of parallel threads")
+
+args = parser.parse_args()
+process_arguments = vars(args)
+pos = str(process_arguments['position'])
+mode = str(process_arguments['mode'])
+n_threads = int(process_arguments['threads'])
+
+if not os.path.exists(pos+"output"):
+	os.mkdir(pos+"output")
+
+if not os.path.exists(pos+os.sep.join(["output","tables"])):
+	os.mkdir(pos+os.sep.join(["output","tables"]))
+
+if mode.lower()=="target" or mode.lower()=="targets":
+	label_folder = "labels_targets"
+	instruction_file = os.sep.join(["configs", "tracking_instructions_targets.json"])
+	napari_name = "napari_target_trajectories.npy"
+	table_name = "trajectories_targets.csv"
+
+elif mode.lower()=="effector" or mode.lower()=="effectors":
+	label_folder = "labels_effectors"
+	instruction_file = os.sep.join(["configs","tracking_instructions_effectors.json"])
+	napari_name = "napari_effector_trajectories.npy"
+	table_name = "trajectories_effectors.csv"
+
+# Locate experiment config
+parent1 = Path(pos).parent
+expfolder = parent1.parent
+config = PurePath(expfolder,Path("config.ini"))
+assert os.path.exists(config),'The configuration file for the experiment could not be located. Abort.'
+print("Configuration file: ",config)
+
+# from exp config fetch spatial calib, channel names
+movie_prefix = ConfigSectionMap(config,"MovieSettings")["movie_prefix"]
+spatial_calibration = float(ConfigSectionMap(config,"MovieSettings")["pxtoum"])
+time_calibration = float(ConfigSectionMap(config,"MovieSettings")["frametomin"])
+len_movie = float(ConfigSectionMap(config,"MovieSettings")["len_movie"])
+shape_x = int(ConfigSectionMap(config,"MovieSettings")["shape_x"])
+shape_y = int(ConfigSectionMap(config,"MovieSettings")["shape_y"])
+
+channel_names, channel_indices = extract_experiment_channels(config)
+nbr_channels = len(channel_names)
+
+# from tracking instructions, fetch btrack config, features, haralick, clean_traj, idea: fetch custom timeline?
+instr_path = PurePath(expfolder,Path(f"{instruction_file}"))
+if os.path.exists(instr_path):
+	print(f"Tracking instructions for the {mode} population has been successfully located.")
+	with open(instr_path, 'r') as f:
+		instructions = json.load(f)
+		print("Reading the following instructions: ",instructions)
+	btrack_config = interpret_tracking_configuration(instructions['btrack_config_path'])
+
+	if 'features' in instructions:
+		features = instructions['features']
+	else:
+		features = None
+	
+	if 'mask_channels' in instructions:
+		mask_channels = instructions['mask_channels']
+	else:
+		mask_channels = None
+	
+	if 'haralick_options' in instructions:
+		haralick_options = instructions['haralick_options']
+	else:
+		haralick_options = None
+
+	if 'post_processing_options' in instructions:
+		post_processing_options = instructions['post_processing_options']
+	else:
+		post_processing_options = None
+else:
+	print('No tracking instructions found. Use standard bTrack motion model.')
+	btrack_config = interpret_tracking_configuration(None)
+	features = None
+	mask_channels = None
+	haralick_options = None
+	post_processing_options = None
+
+if features is None:
+	features = []
+
+# from pos fetch labels
+label_path = natsorted(glob(pos+f"{label_folder}"+os.sep+"*.tif"))
+if len(label_path)>0:
+	print(f"Found {len(label_path)} segmented frames...")
+else:
+	print(f"No segmented frames have been found. Please run segmentation first, skipping...")
+	os.abort()
+
+# Do this if features or Haralick is not None, else don't need stack
+try:
+	file = glob(pos+os.sep.join(["movie", f"{movie_prefix}*.tif"]))[0]
+except IndexError:
+	print('Movie could not be found. Check the prefix. If you intended to measure texture or tone, this will not be performed.')
+	file = None
+	haralick_option = None
+	features = drop_tonal_features(features)
+
+len_movie_auto = auto_load_number_of_frames(file)
+if len_movie_auto is not None:
+	len_movie = len_movie_auto
+
+img_num_channels = _get_img_num_per_channel(channel_indices, len_movie, nbr_channels)
+
+#######################################
+# Loop over all frames and find objects
+#######################################
+
+timestep_dataframes = []
+
+def measure_index(indices):
+	for t in tqdm(indices,desc="frame"):
+		
+		# Load channels at time t
+		img = load_frames(img_num_channels[:,t], file, scale=None, normalize_input=False)
+		lbl = imread(label_path[t])
+
+		df_props = measure_features(img, lbl, features = features+['centroid'], border_dist=None, 
+										channels=channel_names, haralick_options=haralick_options, verbose=False, 
+									)
+		df_props.rename(columns={'centroid-1': 'x', 'centroid-0': 'y'},inplace=True)
+		df_props['t'] = int(t)
+		timestep_dataframes.append(df_props)
+
+# Multithreading
+indices = list(range(img_num_channels.shape[1]))
+chunks = np.array_split(indices, n_threads)
+threads = []
+for i in range(n_threads):
+	thread_i = threading.Thread(target=measure_index, args=[chunks[i]])
+	threads.append(thread_i)
+for th in threads:
+	th.start()
+for th in threads:
+	th.join()
+
+df = pd.concat(timestep_dataframes)	
+df.reset_index(inplace=True, drop=True)
+
+if mask_channels is not None:
+	cols_to_drop = []
+	for mc in mask_channels:
+		columns = df.columns
+		col_contains = [mc in c for c in columns]
+		to_remove = np.array(columns)[np.array(col_contains)]
+		cols_to_drop.extend(to_remove)
+	if len(cols_to_drop)>0:
+		df = df.drop(cols_to_drop, axis=1)
+
+# do tracking
+trajectories, napari_data = track(None,
+					configuration=btrack_config,
+					objects=df, 
+					spatial_calibration=spatial_calibration, 
+					channel_names=channel_names,
+					return_napari_data=True,
+		  			optimizer_options = {'tm_lim': int(12e4)}, 
+		  			track_kwargs={'step_size': 100}, 
+		  			clean_trajectories_kwargs=post_processing_options, 
+		  			volume=(shape_x, shape_y),
+		  			)
+print(trajectories)
+print(trajectories.columns)
+
+# out trajectory table, create POSITION_X_um, POSITION_Y_um, TIME_min (new ones)
+# Save napari data
+np.save(pos+os.sep.join(['output', 'tables', napari_name]), napari_data, allow_pickle=True)
+print(f"napari data successfully saved in {pos+os.sep.join(['output', 'tables'])}")
+
+trajectories.to_csv(pos+os.sep.join(['output', 'tables', table_name]), index=False)
+print(f"Table {table_name} successfully saved in {os.sep.join(['output', 'tables'])}")
+
+del trajectories; del napari_data;
+gc.collect()

celldetective/scripts/train_segmentation_model.py

@@ -0,0 +1,246 @@
+"""
+Copright © 2023 Laboratoire Adhesion et Inflammation, Authored by Remy Torro.
+"""
+
+import argparse
+import os
+import shutil
+from glob import glob
+import json
+from tqdm import tqdm
+import numpy as np
+import random
+
+from celldetective.utils import load_image_dataset, normalize_per_channel, augmenter
+from stardist import fill_label_holes
+from art import tprint
+import matplotlib.pyplot as plt
+
+def interpolate_nan(array_like):
+	array = array_like.copy()
+	
+	isnan_array = ~np.isnan(array)
+	
+	xp = isnan_array.ravel().nonzero()[0]
+	
+	fp = array[~np.isnan(array)]
+	x = np.isnan(array).ravel().nonzero()[0]
+	
+	array[np.isnan(array)] = np.interp(x, xp, fp)
+	
+	return array
+
+tprint("Train")
+
+
+parser = argparse.ArgumentParser(description="Train a signal model from instructions.",
+								formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-c',"--config", required=True,help="Training instructions")
+parser.add_argument('-g',"--use_gpu", required=True, help="Use GPU")
+
+args = parser.parse_args()
+process_arguments = vars(args)
+instructions = str(process_arguments['config'])
+use_gpu = bool(process_arguments['use_gpu'])
+
+if os.path.exists(instructions):
+	with open(instructions, 'r') as f:
+		training_instructions = json.load(f)
+else:
+	print('Training instructions could not be found. Abort.')
+	os.abort()
+
+model_name = training_instructions['model_name']
+target_directory = training_instructions['target_directory']
+model_type = training_instructions['model_type']
+pretrained = training_instructions['pretrained']
+
+datasets = training_instructions['ds']
+
+target_channels = training_instructions['channel_option']
+normalization_percentile = training_instructions['normalization_percentile']
+normalization_clip = training_instructions['normalization_clip']
+normalization_values = training_instructions['normalization_values']
+spatial_calibration = training_instructions['spatial_calibration']
+
+validation_split = training_instructions['validation_split']
+augmentation_factor = training_instructions['augmentation_factor']
+
+learning_rate = training_instructions['learning_rate']
+epochs = training_instructions['epochs']
+batch_size = training_instructions['batch_size']
+
+
+# Load dataset
+print(f'Datasets: {datasets}')
+X,Y = load_image_dataset(datasets, target_channels, train_spatial_calibration=spatial_calibration,
+						mask_suffix='labelled')
+print('Dataset loaded...')
+
+# Normalize images
+X = normalize_per_channel(X,
+						  normalization_percentile_mode=normalization_percentile, 
+						  normalization_values=normalization_values, 
+						  normalization_clipping=normalization_clip
+						  )
+
+for x in X:
+	plt.imshow(x[:,:,0])
+	plt.xlim(0,1004)
+	plt.ylim(0,1002)
+	plt.colorbar()
+	plt.pause(2)
+	plt.close()
+	print(x.shape)
+	interp = interpolate_nan(x)
+	print(interp.shape)
+	print(np.any(np.isnan(x).flatten()))
+	print(np.any(np.isnan(interp).flatten()))
+
+
+Y = [fill_label_holes(y) for y in tqdm(Y)]
+
+assert len(X) > 1, "not enough training data"
+rng = np.random.RandomState()
+ind = rng.permutation(len(X))
+n_val = max(1, int(round(validation_split * len(ind))))
+ind_train, ind_val = ind[:-n_val], ind[-n_val:]
+X_val, Y_val = [X[i] for i in ind_val]  , [Y[i] for i in ind_val]
+X_trn, Y_trn = [X[i] for i in ind_train], [Y[i] for i in ind_train] 
+print('number of images: %3d' % len(X))
+print('- training:       %3d' % len(X_trn))
+print('- validation:     %3d' % len(X_val))
+
+if model_type=='cellpose':
+
+	# do augmentation in place
+	X_aug = []; Y_aug = [];
+	n_val = max(1, int(round(augmentation_factor * len(X_trn))))
+	indices = random.choices(list(np.arange(len(X_trn))), k=n_val)
+	print('Performing image augmentation pre-training...')
+	for i in tqdm(indices):
+		x_aug,y_aug = augmenter(X_trn[i], Y_trn[i])
+		X_aug.append(x_aug)
+		Y_aug.append(y_aug)
+	
+	# Channel axis in front for cellpose
+	X_aug = [np.moveaxis(x,-1,0) for x in X_aug]
+	X_val = [np.moveaxis(x,-1,0) for x in X_val]
+	print('number of augmented images: %3d' % len(X_aug))
+
+	from cellpose.models import CellposeModel
+	from cellpose.io import logger_setup
+	import torch
+	
+	if not use_gpu:
+		device = torch.device("cpu")
+		
+	logger, log_file = logger_setup()
+	print(f'Pretrained model: ',pretrained)
+	if pretrained is not None:
+		pretrained_path = os.sep.join([pretrained,os.path.split(pretrained)[-1]])
+	else:
+		pretrained_path = pretrained
+	
+	model = CellposeModel(gpu=use_gpu, model_type=None, pretrained_model=pretrained_path, diam_mean=30.0, nchan=X_aug[0].shape[0],) 
+	model.train(train_data=X_aug, train_labels=Y_aug, normalize=False, channels=None, batch_size=batch_size,
+				min_train_masks=1,save_path=target_directory+os.sep+model_name,n_epochs=epochs, model_name=model_name, learning_rate=learning_rate, test_data = X_val, test_labels=Y_val)
+
+	file_to_move = glob(os.sep.join([target_directory, model_name, 'models','*']))[0]
+	shutil.move(file_to_move, os.sep.join([target_directory, model_name,''])+os.path.split(file_to_move)[-1])
+	os.rmdir(os.sep.join([target_directory, model_name, 'models']))
+
+	diameter = model.diam_labels
+
+	if pretrained is not None and os.path.split(pretrained)[-1]=='CP_nuclei':
+		standard_diameter = 17.0
+	else:
+		standard_diameter = 30.0
+
+	input_spatial_calibration = spatial_calibration #*diameter / standard_diameter
+
+	config_inputs = {"channels": target_channels, "diameter": standard_diameter, 'cellprob_threshold': 0., 'flow_threshold': 0.4,
+	'normalization_percentile': normalization_percentile, 'normalization_clip': normalization_clip,
+	'normalization_values': normalization_values, 'model_type': 'cellpose',
+	'spatial_calibration': input_spatial_calibration}
+	json_input_config = json.dumps(config_inputs, indent=4)
+	with open(os.sep.join([target_directory, model_name, "config_input.json"]), "w") as outfile:
+		outfile.write(json_input_config)
+
+elif model_type=='stardist':
+	
+	from stardist import calculate_extents, gputools_available
+	from stardist.models import Config2D, StarDist2D
+	
+	n_rays = 32
+	print(gputools_available())
+	
+	n_channel=X_trn[0].shape[-1]
+	
+	# Predict on subsampled grid for increased efficiency and larger field of view
+	grid = (2,2)
+	conf = Config2D (
+		n_rays       = n_rays,
+		grid         = grid,
+		use_gpu      = use_gpu,
+		n_channel_in = n_channel,
+		unet_dropout = 0.0,
+		unet_batch_norm = False,
+		unet_n_conv_per_depth=2,
+		train_learning_rate = learning_rate,
+		train_patch_size = (256,256),
+		train_epochs = epochs,
+		#train_foreground_only=0.9,
+		train_loss_weights=(1,0.2),
+		train_reduce_lr = {'factor': 0.1, 'patience': 30, 'min_delta': 0},
+		unet_n_depth = 3,
+		train_batch_size = batch_size,
+	)
+	
+	if use_gpu:
+		from csbdeep.utils.tf import limit_gpu_memory
+		limit_gpu_memory(None, allow_growth=True)
+
+	if pretrained is None:
+		model = StarDist2D(conf, name=model_name, basedir=target_directory)
+	else:
+		# files_to_copy = glob(os.sep.join([pretrained, '*']))
+		# for f in files_to_copy:
+		# 	shutil.copy(f, os.sep.join([target_directory, model_name, os.path.split(f)[-1]]))
+		idx=1
+		while os.path.exists(os.sep.join([target_directory, model_name])):
+			model_name =  model_name+f'_{idx}'
+			idx+=1
+
+		shutil.copytree(pretrained, os.sep.join([target_directory, model_name]))
+		model = StarDist2D(None, name=model_name, basedir=target_directory)
+		model.config.train_epochs = epochs
+		model.config.train_batch_size = min(len(X_trn),batch_size)
+		model.config.train_learning_rate = learning_rate
+
+	median_size = calculate_extents(list(Y_trn), np.mean)
+	fov = np.array(model._axes_tile_overlap('YX'))
+	print(f"median object size:      {median_size}")
+	print(f"network field of view :  {fov}")
+	if any(median_size > fov):
+		print("WARNING: median object size larger than field of view of the neural network.")
+
+	if augmentation_factor==1.0:
+		model.train(X_trn, Y_trn, validation_data=(X_val,Y_val))
+	else:
+		model.train(X_trn, Y_trn, validation_data=(X_val,Y_val), augmenter=augmenter)
+	model.optimize_thresholds(X_val,Y_val)
+
+	config_inputs = {"channels": target_channels, 'normalization_percentile': normalization_percentile,
+	'normalization_clip': normalization_clip, 'normalization_values': normalization_values, 
+	'model_type': 'stardist', 'spatial_calibration': spatial_calibration}
+
+	json_input_config = json.dumps(config_inputs, indent=4)
+	with open(os.sep.join([target_directory, model_name, "config_input.json"]), "w") as outfile:
+		outfile.write(json_input_config)
+
+print('Done.')
+
+
+
+

celldetective/scripts/train_signal_model.py

@@ -0,0 +1,49 @@
+"""
+Copright © 2023 Laboratoire Adhesion et Inflammation, Authored by Remy Torro.
+"""
+
+import argparse
+import os
+import json
+from pathlib import Path, PurePath
+from glob import glob
+from tqdm import tqdm
+import numpy as np
+import gc
+from art import tprint
+from celldetective.signals import SignalDetectionModel
+
+tprint("Train")
+
+parser = argparse.ArgumentParser(description="Train a signal model from instructions.",
+								formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-c',"--config", required=True,help="Training instructions")
+
+args = parser.parse_args()
+process_arguments = vars(args)
+instructions = str(process_arguments['config'])
+
+if os.path.exists(instructions):
+	with open(instructions, 'r') as f:
+		threshold_instructions = json.load(f)
+		threshold_instructions.update({'n_channels': len(threshold_instructions['channel_option'])})
+		if threshold_instructions['augmentation_factor']>1.0:
+			threshold_instructions.update({'augment': True})
+		else:
+			threshold_instructions.update({'augment': False})
+		threshold_instructions.update({'test_split': 0.})
+else:
+	print('The configuration path is not valid. Abort.')
+	os.abort()
+
+
+model_params = {k:threshold_instructions[k] for k in ('pretrained', 'model_signal_length', 'channel_option', 'n_channels', 'label') if k in threshold_instructions}
+train_params = {k:threshold_instructions[k] for k in ('model_name', 'target_directory', 'channel_option','recompile_pretrained', 'test_split', 'augment', 'epochs', 'learning_rate', 'batch_size', 'validation_split','normalization_percentile','normalization_values','normalization_clip') if k in threshold_instructions}
+
+print(f'model params {model_params}')
+print(f'train params {train_params}')
+
+model = SignalDetectionModel(**model_params)
+model.fit_from_directory(threshold_instructions['ds'], **train_params)
+
+print('Done.')