celldetective 1.1.1.post3__py3-none-any.whl → 1.2.0__py3-none-any.whl

celldetective/scripts/measure_cells.py

@@ -72,6 +72,8 @@ if os.path.exists(instr_path):
 		print("Reading the following instructions: ", instructions)
 	if 'background_correction' in instructions:
 		background_correction = instructions['background_correction']
+	else:
+		background_correction = None
 
 	if 'features' in instructions:
 		features = instructions['features']
@@ -145,14 +147,19 @@ except IndexError:
 # Load trajectories, add centroid if not in trajectory
 trajectories = pos+os.sep.join(['output','tables', table_name])
 if os.path.exists(trajectories):
+	print('trajectory exists...')
 	trajectories = pd.read_csv(trajectories)
 	if 'TRACK_ID' not in list(trajectories.columns):
 		do_iso_intensities = False
 		intensity_measurement_radii = None
 		if clear_previous:
-			trajectories = remove_trajectory_measurements(trajectories, column_labels)
+			print('No TRACK_ID... Clear previous measurements...')
+			trajectories = None #remove_trajectory_measurements(trajectories, column_labels)
+			do_features = True
+			features += ['centroid']
 	else:
 		if clear_previous:
+			print('TRACK_ID found... Clear previous measurements...')
 			trajectories = remove_trajectory_measurements(trajectories, column_labels)
 else:
 	trajectories = None
@@ -161,11 +168,11 @@ else:
 	do_iso_intensities = False
 
 
-if (features is not None) and (trajectories is not None):
-	features = remove_redundant_features(features, 
-										trajectories.columns,
-										channel_names=channel_names
-										)
+# if (features is not None) and (trajectories is not None):
+# 	features = remove_redundant_features(features, 
+# 										trajectories.columns,
+# 										channel_names=channel_names
+# 										)
 
 len_movie_auto = auto_load_number_of_frames(file)
 if len_movie_auto is not None:
@@ -187,6 +194,7 @@ if label_path is None:
 else:
 	do_features = True
 
+
 #######################################
 # Loop over all frames and find objects
 #######################################
@@ -196,6 +204,9 @@ if trajectories is None:
 	print('Use features as a substitute for the trajectory table.')
 	if 'label' not in features:
 		features.append('label')
+
+
+
 features_log=f'features: {features}'
 border_distances_log=f'border_distances: {border_distances}'
 haralick_options_log=f'haralick_options: {haralick_options}'
@@ -237,7 +248,7 @@ def measure_index(indices):
 				column_labels = {'track': "ID", 'time': column_labels['time'], 'x': column_labels['x'],
 								 'y': column_labels['y']}
 			feature_table.rename(columns={'centroid-1': 'POSITION_X', 'centroid-0': 'POSITION_Y'}, inplace=True)
-
+		
 		if do_iso_intensities:
 			iso_table = measure_isotropic_intensity(positions_at_t, img, channels=channel_names, intensity_measurement_radii=intensity_measurement_radii, column_labels=column_labels, operations=isotropic_operations, verbose=False)
 
@@ -249,11 +260,20 @@ def measure_index(indices):
 		elif do_features:
 			measurements_at_t = positions_at_t.merge(feature_table, how='outer', on='class_id',suffixes=('', '_delme'))
 			measurements_at_t = measurements_at_t[[c for c in measurements_at_t.columns if not c.endswith('_delme')]]
-			
+	
+		center_of_mass_x_cols = [c for c in list(measurements_at_t.columns) if c.endswith('centre_of_mass_x')]
+		center_of_mass_y_cols = [c for c in list(measurements_at_t.columns) if c.endswith('centre_of_mass_y')]
+		for c in center_of_mass_x_cols:
+			measurements_at_t.loc[:,c.replace('_x','_POSITION_X')] = measurements_at_t[c] + measurements_at_t['POSITION_X']
+		for c in center_of_mass_y_cols:
+			measurements_at_t.loc[:,c.replace('_y','_POSITION_Y')] = measurements_at_t[c] + measurements_at_t['POSITION_Y']
+		measurements_at_t = measurements_at_t.drop(columns = center_of_mass_x_cols+center_of_mass_y_cols)
+		
 		if measurements_at_t is not None:
 			measurements_at_t[column_labels['time']] = t
 			timestep_dataframes.append(measurements_at_t)
 
+
 # Multithreading
 indices = list(range(img_num_channels.shape[1]))
 chunks = np.array_split(indices, n_threads)

celldetective/scripts/measure_relative.py

@@ -0,0 +1,103 @@
+import argparse
+import os
+import json
+from celldetective.relative_measurements import measure_pair_signals_at_position, update_effector_table, extract_neighborhoods_from_pickles
+from celldetective.utils import ConfigSectionMap, extract_experiment_channels
+
+from pathlib import Path, PurePath
+
+import pandas as pd
+
+from art import tprint
+
+
+tprint("Measure pairs")
+
+parser = argparse.ArgumentParser(description="Measure features and intensities in a multichannel timeseries.",
+								 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-p', "--position", required=True, help="Path to the position")
+
+args = parser.parse_args()
+process_arguments = vars(args)
+pos = str(process_arguments['position'])
+
+instruction_file = os.sep.join(['configs', "neighborhood_instructions.json"])
+
+# Locate experiment config
+parent1 = Path(pos).parent
+expfolder = parent1.parent
+config = PurePath(expfolder, Path("config.ini"))
+assert os.path.exists(config), 'The configuration file for the experiment could not be located. Abort.'
+print("Configuration file: ", config)
+
+# from exp config fetch spatial calib, channel names
+movie_prefix = ConfigSectionMap(config, "MovieSettings")["movie_prefix"]
+spatial_calibration = float(ConfigSectionMap(config, "MovieSettings")["pxtoum"])
+time_calibration = float(ConfigSectionMap(config, "MovieSettings")["frametomin"])
+len_movie = float(ConfigSectionMap(config, "MovieSettings")["len_movie"])
+channel_names, channel_inneigh_protocoles = extract_experiment_channels(config)
+nbr_channels = len(channel_names)
+
+# from tracking instructions, fetch btrack config, features, haralick, clean_traj, idea: fetch custom timeline?
+instr_path = PurePath(expfolder, Path(f"{instruction_file}"))
+previous_pair_table_path = pos + os.sep.join(['output', 'tables', 'trajectories_pairs.csv'])
+
+# if os.path.exists(instr_path):
+# 	print(f"Neighborhood instructions has been successfully located.")
+# 	with open(instr_path, 'r') as f:
+# 		instructions = json.load(f)
+# 		print("Reading the following instructions: ", instructions)
+
+# 	if 'distance' in instructions:
+# 		distance = instructions['distance'][0]
+# 	else:
+# 		distance = None
+# else:
+# 	print('No measurement instructions found')
+# 	os.abort()
+
+previous_neighborhoods = []
+associated_reference_population = []
+
+# if distance is None:
+# 	print('No measurement could be performed. Check your inputs.')
+# 	print('Done.')
+# 	os.abort()
+# 	#distance = 0
+# else:
+neighborhoods_to_measure = extract_neighborhoods_from_pickles(pos)
+all_df_pairs = []
+if os.path.exists(previous_pair_table_path):
+	df_0 = pd.read_csv(previous_pair_table_path)
+	previous_neighborhoods = [c.replace('status_','') for c in list(df_0.columns) if c.startswith('status_neighborhood')]
+	for n in previous_neighborhoods:
+		associated_reference_population.append(df_0.loc[~df_0['status_'+n].isnull(),'reference_population'].values[0])
+	print(f'{previous_neighborhoods=} {associated_reference_population=}')
+	all_df_pairs.append(df_0)
+for k,neigh_protocol in enumerate(neighborhoods_to_measure):
+	if neigh_protocol['description'] not in previous_neighborhoods:
+		df_pairs = measure_pair_signals_at_position(pos, neigh_protocol)
+		print(f'{df_pairs=}')
+		if 'REFERENCE_ID' in list(df_pairs.columns):
+			all_df_pairs.append(df_pairs)
+	elif neigh_protocol['description'] in previous_neighborhoods and neigh_protocol['reference'] != associated_reference_population[previous_neighborhoods.index(neigh_protocol['description'])]:
+		df_pairs = measure_pair_signals_at_position(pos, neigh_protocol)
+		if 'REFERENCE_ID' in list(df_pairs.columns):
+			all_df_pairs.append(df_pairs)			
+
+print(f'{len(all_df_pairs)} neighborhood measurements sets were computed...')
+
+if len(all_df_pairs)>1:
+	print('Merging...')
+	df_pairs = all_df_pairs[0]
+	for i in range(1,len(all_df_pairs)):
+		cols = [c1 for c1,c2 in zip(list(df_pairs.columns), list(all_df_pairs[i].columns)) if c1==c2]
+		df_pairs = pd.merge(df_pairs.round(decimals=6), all_df_pairs[i].round(decimals=6), how="outer", on=cols)
+elif len(all_df_pairs)==1:
+	df_pairs = all_df_pairs[0]
+
+print('Writing table...')
+df_pairs = df_pairs.sort_values(by=['reference_population', 'neighbor_population', 'REFERENCE_ID', 'NEIGHBOR_ID', 'FRAME'])
+df_pairs.to_csv(previous_pair_table_path, index=False)
+print('Done.')
+

celldetective/scripts/segment_cells.py

@@ -129,10 +129,10 @@ img_num_channels = _get_img_num_per_channel(channel_indices, int(len_movie), nbr
 
 # If everything OK, prepare output, load models
 print('Erasing previous segmentation folder.')
-if os.path.exists(os.sep.join([pos,label_folder])):
-	rmtree(os.sep.join([pos,label_folder]))
-os.mkdir(os.sep.join([pos,label_folder]))
-print(f'Folder {os.sep.join([pos,label_folder])} successfully generated.')
+if os.path.exists(pos+label_folder):
+	rmtree(pos+label_folder)
+os.mkdir(pos+label_folder)
+print(f'Folder {pos+label_folder} successfully generated.')
 log=f'segmentation model: {modelname}\n'
 with open(pos+f'log_{mode}.json', 'a') as f:
 	f.write(f'{datetime.datetime.now()} SEGMENT \n')
@@ -203,7 +203,7 @@ def segment_index(indices):
 		if Y_pred.shape != template.shape[:2]:
 			Y_pred = resize(Y_pred, template.shape[:2], order=0)
 
-		save_tiff_imagej_compatible(os.sep.join([pos,label_folder,f"{str(t).zfill(4)}.tif"]), Y_pred, axes='YX')
+		save_tiff_imagej_compatible(pos+os.sep.join([label_folder,f"{str(t).zfill(4)}.tif"]), Y_pred, axes='YX')
 
 		del f;
 		del template;

celldetective/scripts/track_cells.py

@@ -9,7 +9,7 @@ import json
 from celldetective.io import auto_load_number_of_frames, load_frames, interpret_tracking_configuration
 from celldetective.utils import extract_experiment_channels, _extract_channel_indices_from_config, _extract_channel_indices, ConfigSectionMap, _extract_nbr_channels_from_config, _get_img_num_per_channel, extract_experiment_channels
 from celldetective.measure import drop_tonal_features, measure_features
-from celldetective import track
+from celldetective.tracking import track
 from pathlib import Path, PurePath
 from glob import glob
 from shutil import rmtree
@@ -213,5 +213,8 @@ print(f"napari data successfully saved in {pos+os.sep.join(['output', 'tables'])
 trajectories.to_csv(pos+os.sep.join(['output', 'tables', table_name]), index=False)
 print(f"Table {table_name} successfully saved in {os.sep.join(['output', 'tables'])}")
 
+if os.path.exists(pos+os.sep.join(['output', 'tables', table_name.replace('.csv','.pkl')])):
+	os.remove(pos+os.sep.join(['output', 'tables', table_name.replace('.csv','.pkl')]))
+
 del trajectories; del napari_data;
 gc.collect()

celldetective/scripts/train_segmentation_model.py

@@ -16,7 +16,8 @@ from celldetective.io import normalize_multichannel
 from stardist import fill_label_holes
 from art import tprint
 import matplotlib.pyplot as plt
-
+from distutils.dir_util import copy_tree
+from csbdeep.utils import save_json
 
 tprint("Train")
 
@@ -169,22 +170,17 @@ elif model_type=='stardist':
 	
 	# Predict on subsampled grid for increased efficiency and larger field of view
 	grid = (2,2)
-	conf = Config2D (
+	conf = Config2D(
 		n_rays       = n_rays,
 		grid         = grid,
 		use_gpu      = use_gpu,
 		n_channel_in = n_channel,
-		unet_dropout = 0.0,
-		unet_batch_norm = False,
-		unet_n_conv_per_depth=2,
 		train_learning_rate = learning_rate,
 		train_patch_size = (256,256),
 		train_epochs = epochs,
-		#train_foreground_only=0.9,
-		train_loss_weights=(1,0.2),
 		train_reduce_lr = {'factor': 0.1, 'patience': 30, 'min_delta': 0},
-		unet_n_depth = 3,
 		train_batch_size = batch_size,
+		train_steps_per_epoch = int(augmentation_factor*len(X_trn)),
 	)
 	
 	if use_gpu:
@@ -194,19 +190,28 @@ elif model_type=='stardist':
 	if pretrained is None:
 		model = StarDist2D(conf, name=model_name, basedir=target_directory)
 	else:
-		# files_to_copy = glob(os.sep.join([pretrained, '*']))
-		# for f in files_to_copy:
-		# 	shutil.copy(f, os.sep.join([target_directory, model_name, os.path.split(f)[-1]]))
-		idx=1
-		while os.path.exists(os.sep.join([target_directory, model_name])):
-			model_name =  model_name+f'_{idx}'
-			idx+=1
-
-		shutil.copytree(pretrained, os.sep.join([target_directory, model_name]))
+
+		os.rename(instructions, os.sep.join([target_directory, model_name, 'temp.json']))
+		copy_tree(pretrained, os.sep.join([target_directory, model_name]))
+		
+		if os.path.exists(os.sep.join([target_directory, model_name, 'training_instructions.json'])):
+			os.remove(os.sep.join([target_directory, model_name, 'training_instructions.json']))
+		if os.path.exists(os.sep.join([target_directory, model_name, 'config_input.json'])):
+			os.remove(os.sep.join([target_directory, model_name, 'config_input.json']))
+		if os.path.exists(os.sep.join([target_directory, model_name, 'logs'+os.sep])):
+			shutil.rmtree(os.sep.join([target_directory, model_name, 'logs']))
+		os.rename(os.sep.join([target_directory, model_name, 'temp.json']),os.sep.join([target_directory, model_name, 'training_instructions.json']))
+
+		#shutil.copytree(pretrained, os.sep.join([target_directory, model_name]))
 		model = StarDist2D(None, name=model_name, basedir=target_directory)
 		model.config.train_epochs = epochs
 		model.config.train_batch_size = min(len(X_trn),batch_size)
-		model.config.train_learning_rate = learning_rate
+		model.config.train_learning_rate = learning_rate # perf seems bad if lr is changed in transfer
+		model.config.use_gpu = use_gpu
+		model.config.train_reduce_lr = {'factor': 0.1, 'patience': 10, 'min_delta': 0}
+		print(f'{model.config=}')
+
+		save_json(vars(model.config), os.sep.join([target_directory, model_name, 'config.json']))
 
 	median_size = calculate_extents(list(Y_trn), np.mean)
 	fov = np.array(model._axes_tile_overlap('YX'))

celldetective/scripts/train_signal_model.py

@@ -12,6 +12,7 @@ import numpy as np
 import gc
 from art import tprint
 from celldetective.signals import SignalDetectionModel
+from celldetective.io import locate_signal_model
 
 tprint("Train")
 
@@ -36,14 +37,46 @@ else:
 	print('The configuration path is not valid. Abort.')
 	os.abort()
 
+all_classes = []
+for d in threshold_instructions["ds"]:
+	datasets = glob(d+os.sep+"*.npy")
+	for dd in datasets:
+		data = np.load(dd, allow_pickle=True)
+		classes = np.unique([ddd["class"] for ddd in data])
+		all_classes.extend(classes)
+all_classes = np.unique(all_classes)
+print(all_classes,len(all_classes))
+
+n_classes = len(all_classes)
 
 model_params = {k:threshold_instructions[k] for k in ('pretrained', 'model_signal_length', 'channel_option', 'n_channels', 'label') if k in threshold_instructions}
+model_params.update({'n_classes': n_classes})
+
 train_params = {k:threshold_instructions[k] for k in ('model_name', 'target_directory', 'channel_option','recompile_pretrained', 'test_split', 'augment', 'epochs', 'learning_rate', 'batch_size', 'validation_split','normalization_percentile','normalization_values','normalization_clip') if k in threshold_instructions}
 
 print(f'model params {model_params}')
 print(f'train params {train_params}')
 
 model = SignalDetectionModel(**model_params)
+print(threshold_instructions['ds'])
 model.fit_from_directory(threshold_instructions['ds'], **train_params)
 
+
+# if neighborhood of interest in training instructions, write it in config!
+if 'neighborhood_of_interest' in threshold_instructions:
+	if threshold_instructions['neighborhood_of_interest'] is not None:
+		
+		model_path = locate_signal_model(threshold_instructions['model_name'], path=None, pairs=True)
+		complete_path = model_path #+model
+		complete_path = rf"{complete_path}"
+		model_config_path = os.sep.join([complete_path,'config_input.json'])
+		model_config_path = rf"{model_config_path}"
+
+		f = open(model_config_path)
+		config = json.load(f)
+		config.update({'neighborhood_of_interest': threshold_instructions['neighborhood_of_interest'], 'reference_population': threshold_instructions['reference_population'], 'neighbor_population': threshold_instructions['neighbor_population']})
+		json_string = json.dumps(config)
+		with open(model_config_path, 'w') as outfile:
+			outfile.write(json_string)
+
 print('Done.')

celldetective/segmentation.py

@@ -20,6 +20,7 @@ from skimage.segmentation import watershed
 from skimage.feature import peak_local_max
 from skimage.measure import regionprops_table
 from skimage.exposure import match_histograms
+from scipy.ndimage import zoom
 import pandas as pd
 import subprocess
 
@@ -27,7 +28,7 @@ import subprocess
 abs_path = os.sep.join([os.path.split(os.path.dirname(os.path.realpath(__file__)))[0],'celldetective'])
 
 def segment(stack, model_name, channels=None, spatial_calibration=None, view_on_napari=False,
-			use_gpu=True, time_flat_normalization=False, time_flat_percentiles=(0.0,99.99)):
+			use_gpu=True, channel_axis=-1):
 	
 	"""
 	
@@ -85,7 +86,10 @@ def segment(stack, model_name, channels=None, spatial_calibration=None, view_on_
 	if not use_gpu:
 		os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
 	else:
-		os.environ['CUDA_VISIBLE_DEVICES'] = '0'		
+		os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+	if channel_axis != -1:
+		stack = np.moveaxis(stack, channel_axis, -1)
 
 	if channels is not None:
 		assert len(channels)==stack.shape[-1],f'The channel names provided do not match with the expected number of channels in the stack: {stack.shape[-1]}.'
@@ -96,48 +100,83 @@ def segment(stack, model_name, channels=None, spatial_calibration=None, view_on_
 	required_spatial_calibration = input_config['spatial_calibration']
 	model_type = input_config['model_type']
 
-	if 'normalize' in input_config:
-		normalize = input_config['normalize']
-	else:
-		normalize = True
+	normalization_percentile = input_config['normalization_percentile']
+	normalization_clip = input_config['normalization_clip']
+	normalization_values = input_config['normalization_values']
 
 	if model_type=='cellpose':
 		diameter = input_config['diameter']
-		if diameter!=30:
-			required_spatial_calibration = None
+		# if diameter!=30:
+		# 	required_spatial_calibration = None
 		cellprob_threshold = input_config['cellprob_threshold']
 		flow_threshold = input_config['flow_threshold']
 
 	scale = _estimate_scale_factor(spatial_calibration, required_spatial_calibration)
 
 	if model_type=='stardist':
+
 		model = StarDist2D(None, name=model_name, basedir=Path(model_path).parent)
-		print(f"StarDist model {model_name} successfully loaded")
+		model.config.use_gpu = use_gpu
+		model.use_gpu = use_gpu
+		print(f"StarDist model {model_name} successfully loaded.")
+		scale_model = scale
 
 	elif model_type=='cellpose':
-		model = CellposeModel(gpu=use_gpu, pretrained_model=model_path+model_path.split('/')[-2], diam_mean=30.0)
+		
+		import torch
+		if not use_gpu:
+			device = torch.device("cpu")
+		else:
+			device = torch.device("cuda")
+
+		model = CellposeModel(gpu=use_gpu, device=device, pretrained_model=model_path+model_path.split('/')[-2], model_type=None, nchan=len(required_channels))
 		if scale is None:
 			scale_model = model.diam_mean / model.diam_labels
 		else:
 			scale_model = scale * model.diam_mean / model.diam_labels
+		print(f"Diam mean: {model.diam_mean}; Diam labels: {model.diam_labels}; Final rescaling: {scale_model}...")
+		print(f'Cellpose model {model_name} successfully loaded.')
 
 	labels = []
-	if (time_flat_normalization)*normalize:
-		normalization_values = get_stack_normalization_values(stack[:,:,:,channel_indices], percentiles=time_flat_percentiles)
-	else:
-		normalization_values = [None]*len(channel_indices)
 
 	for t in tqdm(range(len(stack)),desc="frame"):
 
 		# normalize
-		frame = stack[t,:,:,np.array(channel_indices)]
-		if np.argmin(frame.shape)!=(frame.ndim-1):
-			frame = np.moveaxis(frame,np.argmin(frame.shape),-1)
-		if normalize:
-			frame = normalize_multichannel(frame, values=normalization_values)
-
-		if scale is not None:
-			frame = [ndi.zoom(frame[:,:,c].copy(), [scale_model,scale_model], order=3, prefilter=False) for c in range(frame.shape[-1])]
+		channel_indices = np.array(channel_indices)
+		none_channel_indices = np.where(channel_indices==None)[0]
+		channel_indices[channel_indices==None] = 0
+		print(channel_indices)
+
+		frame = stack[t,:,:,channel_indices.astype(int)].astype(float)
+		if frame.ndim==2:
+			frame = frame[:,:,np.newaxis]
+		if frame.ndim==3 and np.array(frame.shape).argmin()==0:
+			frame = np.moveaxis(frame,0,-1)
+		template = frame.copy()
+
+		values = []
+		percentiles = []
+		for k in range(len(normalization_percentile)):
+			if normalization_percentile[k]:
+				percentiles.append(normalization_values[k])
+				values.append(None)
+			else:
+				percentiles.append(None)
+				values.append(normalization_values[k])
+
+		frame = normalize_multichannel(frame, **{"percentiles": percentiles, 'values': values, 'clip': normalization_clip})
+		
+		if scale_model is not None:
+			frame = [zoom(frame[:,:,c].copy(), [scale_model,scale_model], order=3, prefilter=False) for c in range(frame.shape[-1])]
+			frame = np.moveaxis(frame,0,-1)
+		
+		for k in range(frame.shape[2]):
+			unique_values = np.unique(frame[:,:,k])
+			if len(unique_values)==1:
+				frame[0,0,k] += 1
+		
+		frame = np.moveaxis([interpolate_nan(frame[:,:,c].copy()) for c in range(frame.shape[-1])],0,-1)
+		frame[:,:,none_channel_indices] = 0.
 
 		if model_type=="stardist":
 
@@ -145,16 +184,15 @@ def segment(stack, model_name, channels=None, spatial_calibration=None, view_on_
 			Y_pred = Y_pred.astype(np.uint16)
 
 		elif model_type=="cellpose":
-
-			Y_pred, _, _ = model.eval(frame, diameter = diameter, cellprob_threshold=cellprob_threshold, flow_threshold=flow_threshold, channels=None, normalize=False)
+			
+			img = np.moveaxis(frame, -1, 0)
+			Y_pred, _, _ = model.eval(img, diameter = diameter, cellprob_threshold=cellprob_threshold, flow_threshold=flow_threshold, channels=None, normalize=False)
 			Y_pred = Y_pred.astype(np.uint16)
 
-		if scale is not None:
-			Y_pred = ndi.zoom(Y_pred, [1./scale_model,1./scale_model],order=0)
-
-
 		if Y_pred.shape != stack[0].shape[:2]:
-			Y_pred = resize(Y_pred, stack[0].shape, order=0)
+			Y_pred = zoom(Y_pred, [1./scale_model,1./scale_model],order=0)
+		if Y_pred.shape != template.shape[:2]:
+			Y_pred = resize(Y_pred, template.shape[:2], order=0)
 
 		labels.append(Y_pred)