celldetective 1.1.1.post4__py3-none-any.whl → 1.2.1__py3-none-any.whl

celldetective/signals.py

@@ -18,8 +18,8 @@ from sklearn.metrics import jaccard_score, balanced_accuracy_score, precision_sc
 from scipy.interpolate import interp1d
 from scipy.ndimage import shift
 
-from celldetective.io import get_signal_models_list, locate_signal_model
-from celldetective.tracking import clean_trajectories
+from celldetective.io import get_signal_models_list, locate_signal_model, get_position_pickle, get_position_table
+from celldetective.tracking import clean_trajectories, interpolate_nan_properties
 from celldetective.utils import regression_plot, train_test_split, compute_weights
 import matplotlib.pyplot as plt
 from natsort import natsorted
@@ -32,6 +32,7 @@ from scipy.optimize import curve_fit
 import time
 import math
 import pandas as pd
+from pandas.api.types import is_numeric_dtype
 
 abs_path = os.sep.join([os.path.split(os.path.dirname(os.path.realpath(__file__)))[0],'celldetective'])
 
@@ -154,6 +155,7 @@ def analyze_signals(trajectories, model, interpolate_na=True,
 	f = open(model_config_path)
 	config = json.load(f)
 	required_signals = config["channels"]
+	model_signal_length = config['model_signal_length']
 
 	try:
 		label = config['label']
@@ -189,6 +191,8 @@ def analyze_signals(trajectories, model, interpolate_na=True,
 	trajectories_clean = clean_trajectories(trajectories, interpolate_na=interpolate_na, interpolate_position_gaps=interpolate_na, column_labels=column_labels)
 
 	max_signal_size = int(trajectories_clean[column_labels['time']].max()) + 2
+	assert max_signal_size <= model_signal_length,f'The current signals are longer ({max_signal_size}) than the maximum expected input ({model_signal_length}) for this signal analysis model. Abort...'
+
 	tracks = trajectories_clean[column_labels['track']].unique()
 	signals = np.zeros((len(tracks),max_signal_size, len(selected_signals)))
 
@@ -334,6 +338,392 @@ def analyze_signals_at_position(pos, model, mode, use_gpu=True, return_table=Fal
 	else:
 		return None		
 
+def analyze_pair_signals_at_position(pos, model, use_gpu=True):
+	
+	"""
+
+	"""
+
+	pos = pos.replace('\\','/')
+	pos = rf"{pos}"
+	assert os.path.exists(pos),f'Position {pos} is not a valid path.'
+	if not pos.endswith('/'):
+		pos += '/'
+					
+	df_targets = get_position_pickle(pos, population='targets')
+	df_effectors = get_position_pickle(pos, population='effectors')
+	dataframes = {
+		'targets': df_targets,
+		'effectors': df_effectors,
+	}
+	df_pairs = get_position_table(pos, population='pairs')
+
+	# Need to identify expected reference / neighbor tables
+	model_path = locate_signal_model(model, pairs=True)
+	print(f'Looking for model in {model_path}...')
+	complete_path = model_path
+	complete_path = rf"{complete_path}"
+	model_config_path = os.sep.join([complete_path, 'config_input.json'])
+	model_config_path = rf"{model_config_path}"
+	f = open(model_config_path)
+	model_config_path = json.load(f)
+
+	reference_population = model_config_path['reference_population']
+	neighbor_population = model_config_path['neighbor_population']
+
+	df = analyze_pair_signals(df_pairs, dataframes[reference_population], dataframes[neighbor_population], model=model)
+	
+	table = pos + os.sep.join(["output","tables",f"trajectories_pairs.csv"])
+	df.to_csv(table, index=False)
+
+	return None		
+
+
+def analyze_signals(trajectories, model, interpolate_na=True,
+                    selected_signals=None,
+                    model_path=None,
+                    column_labels={'track': "TRACK_ID", 'time': 'FRAME', 'x': 'POSITION_X', 'y': 'POSITION_Y'},
+                    plot_outcome=False, output_dir=None):
+    """
+	Analyzes signals from trajectory data using a specified signal detection model and configuration.
+
+	This function preprocesses trajectory data, selects specified signals, and applies a pretrained signal detection
+	model to predict classes and times of interest for each trajectory. It supports custom column labeling, interpolation
+	of missing values, and plotting of analysis outcomes.
+
+	Parameters
+	----------
+	trajectories : pandas.DataFrame
+		DataFrame containing trajectory data with columns for track ID, frame, position, and signals.
+	model : str
+		The name of the signal detection model to be used for analysis.
+	interpolate_na : bool, optional
+		Whether to interpolate missing values in the trajectories (default is True).
+	selected_signals : list of str, optional
+		A list of column names from `trajectories` representing the signals to be analyzed. If None, signals will
+		be automatically selected based on the model configuration (default is None).
+	column_labels : dict, optional
+		A dictionary mapping the default column names ('track', 'time', 'x', 'y') to the corresponding column names
+		in `trajectories` (default is {'track': "TRACK_ID", 'time': 'FRAME', 'x': 'POSITION_X', 'y': 'POSITION_Y'}).
+	plot_outcome : bool, optional
+		If True, generates and saves a plot of the signal analysis outcome (default is False).
+	output_dir : str, optional
+		The directory where the outcome plot will be saved. Required if `plot_outcome` is True (default is None).
+
+	Returns
+	-------
+	pandas.DataFrame
+		The input `trajectories` DataFrame with additional columns for predicted classes, times of interest, and
+		corresponding colors based on status and class.
+
+	Raises
+	------
+	AssertionError
+		If the model or its configuration file cannot be located.
+
+	Notes
+	-----
+	- The function relies on an external model configuration file (`config_input.json`) located in the model's directory.
+	- Signal selection and preprocessing are based on the requirements specified in the model's configuration.
+
+	"""
+
+    model_path = locate_signal_model(model, path=model_path)
+    complete_path = model_path  # +model
+    complete_path = rf"{complete_path}"
+    model_config_path = os.sep.join([complete_path, 'config_input.json'])
+    model_config_path = rf"{model_config_path}"
+    assert os.path.exists(complete_path), f'Model {model} could not be located in folder {model_path}... Abort.'
+    assert os.path.exists(
+        model_config_path), f'Model configuration could not be located in folder {model_path}... Abort.'
+
+    available_signals = list(trajectories.columns)
+    print('The available_signals are : ', available_signals)
+
+    f = open(model_config_path)
+    config = json.load(f)
+    required_signals = config["channels"]
+
+    try:
+        label = config['label']
+        if label == '':
+            label = None
+    except:
+        label = None
+
+    if selected_signals is None:
+        selected_signals = []
+        for s in required_signals:
+            pattern_test = [s in a or s == a for a in available_signals]
+            print(f'Pattern test for signal {s}: ', pattern_test)
+            assert np.any(
+                pattern_test), f'No signal matches with the requirements of the model {required_signals}. Please pass the signals manually with the argument selected_signals or add measurements. Abort.'
+            valid_columns = np.array(available_signals)[np.array(pattern_test)]
+            if len(valid_columns) == 1:
+                selected_signals.append(valid_columns[0])
+            else:
+                # print(test_number_of_nan(trajectories, valid_columns))
+                print(f'Found several candidate signals: {valid_columns}')
+                for vc in natsorted(valid_columns):
+                    if 'circle' in vc:
+                        selected_signals.append(vc)
+                        break
+                else:
+                    selected_signals.append(valid_columns[0])
+        # do something more complicated in case of one to many columns
+        # pass
+    else:
+        assert len(selected_signals) == len(
+            required_signals), f'Mismatch between the number of required signals {required_signals} and the provided signals {selected_signals}... Abort.'
+
+    print(f'The following channels will be passed to the model: {selected_signals}')
+    trajectories_clean = clean_trajectories(trajectories, interpolate_na=interpolate_na,
+                                            interpolate_position_gaps=interpolate_na, column_labels=column_labels)
+
+    max_signal_size = int(trajectories_clean[column_labels['time']].max()) + 2
+    tracks = trajectories_clean[column_labels['track']].unique()
+    signals = np.zeros((len(tracks), max_signal_size, len(selected_signals)))
+
+    for i, (tid, group) in enumerate(trajectories_clean.groupby(column_labels['track'])):
+        frames = group[column_labels['time']].to_numpy().astype(int)
+        for j, col in enumerate(selected_signals):
+            signal = group[col].to_numpy()
+            signals[i, frames, j] = signal
+            signals[i, max(frames):, j] = signal[-1]
+
+    # for i in range(5):
+    # 	print('pre model')
+    # 	plt.plot(signals[i,:,0])
+    # 	plt.show()
+
+    model = SignalDetectionModel(pretrained=complete_path)
+    print('signal shape: ', signals.shape)
+
+    classes = model.predict_class(signals)
+    times_recast = model.predict_time_of_interest(signals)
+
+    if label is None:
+        class_col = 'class'
+        time_col = 't0'
+        status_col = 'status'
+    else:
+        class_col = 'class_' + label
+        time_col = 't_' + label
+        status_col = 'status_' + label
+
+    for i, (tid, group) in enumerate(trajectories.groupby(column_labels['track'])):
+        indices = group.index
+        trajectories.loc[indices, class_col] = classes[i]
+        trajectories.loc[indices, time_col] = times_recast[i]
+    print('Done.')
+
+    for tid, group in trajectories.groupby(column_labels['track']):
+
+        indices = group.index
+        t0 = group[time_col].to_numpy()[0]
+        cclass = group[class_col].to_numpy()[0]
+        timeline = group[column_labels['time']].to_numpy()
+        status = np.zeros_like(timeline)
+        if t0 > 0:
+            status[timeline >= t0] = 1.
+        if cclass == 2:
+            status[:] = 2
+        if cclass > 2:
+            status[:] = 42
+        status_color = [color_from_status(s) for s in status]
+        class_color = [color_from_class(cclass) for i in range(len(status))]
+
+        trajectories.loc[indices, status_col] = status
+        trajectories.loc[indices, 'status_color'] = status_color
+        trajectories.loc[indices, 'class_color'] = class_color
+
+    if plot_outcome:
+        fig, ax = plt.subplots(1, len(selected_signals), figsize=(10, 5))
+        for i, s in enumerate(selected_signals):
+            for k, (tid, group) in enumerate(trajectories.groupby(column_labels['track'])):
+                cclass = group[class_col].to_numpy()[0]
+                t0 = group[time_col].to_numpy()[0]
+                timeline = group[column_labels['time']].to_numpy()
+                if cclass == 0:
+                    if len(selected_signals) > 1:
+                        ax[i].plot(timeline - t0, group[s].to_numpy(), c='tab:blue', alpha=0.1)
+                    else:
+                        ax.plot(timeline - t0, group[s].to_numpy(), c='tab:blue', alpha=0.1)
+        if len(selected_signals) > 1:
+            for a, s in zip(ax, selected_signals):
+                a.set_title(s)
+                a.set_xlabel(r'time - t$_0$ [frame]')
+                a.spines['top'].set_visible(False)
+                a.spines['right'].set_visible(False)
+        else:
+            ax.set_title(s)
+            ax.set_xlabel(r'time - t$_0$ [frame]')
+            ax.spines['top'].set_visible(False)
+            ax.spines['right'].set_visible(False)
+        plt.tight_layout()
+        if output_dir is not None:
+            plt.savefig(output_dir + 'signal_collapse.png', bbox_inches='tight', dpi=300)
+        plt.pause(3)
+        plt.close()
+
+    return trajectories
+
+
+def analyze_pair_signals(trajectories_pairs,trajectories_reference,trajectories_neighbors, model, interpolate_na=True, selected_signals=None,
+						model_path=None, plot_outcome=False, output_dir=None, column_labels = {'track': "TRACK_ID", 'time': 'FRAME', 'x': 'POSITION_X', 'y': 'POSITION_Y'}):
+	"""
+	"""
+
+	model_path = locate_signal_model(model, path=model_path, pairs=True)
+	print(f'Looking for model in {model_path}...')
+	complete_path = model_path
+	complete_path = rf"{complete_path}"
+	model_config_path = os.sep.join([complete_path, 'config_input.json'])
+	model_config_path = rf"{model_config_path}"
+	assert os.path.exists(complete_path), f'Model {model} could not be located in folder {model_path}... Abort.'
+	assert os.path.exists(model_config_path), f'Model configuration could not be located in folder {model_path}... Abort.'
+
+	trajectories_pairs = trajectories_pairs.rename(columns=lambda x: 'pair_' + x)
+	trajectories_reference = trajectories_reference.rename(columns=lambda x: 'reference_' + x)
+	trajectories_neighbors = trajectories_neighbors.rename(columns=lambda x: 'neighbor_' + x)
+
+	if 'pair_position' in list(trajectories_pairs.columns):
+		pair_groupby_cols = ['pair_position', 'pair_REFERENCE_ID', 'pair_NEIGHBOR_ID']
+	else:
+		pair_groupby_cols = ['pair_REFERENCE_ID', 'pair_NEIGHBOR_ID']
+
+	if 'reference_position' in list(trajectories_reference.columns):
+		reference_groupby_cols = ['reference_position', 'reference_TRACK_ID']
+	else:
+		reference_groupby_cols = ['reference_TRACK_ID']
+
+	if 'neighbor_position' in list(trajectories_neighbors.columns):
+		neighbor_groupby_cols = ['neighbor_position', 'neighbor_TRACK_ID']
+	else:
+		neighbor_groupby_cols = ['neighbor_TRACK_ID']
+
+	available_signals = [] #list(trajectories_pairs.columns) + list(trajectories_reference.columns) + list(trajectories_neighbors.columns)
+	for col in list(trajectories_pairs.columns):
+		if is_numeric_dtype(trajectories_pairs[col]):
+			available_signals.append(col)
+	for col in list(trajectories_reference.columns):
+		if is_numeric_dtype(trajectories_reference[col]):
+			available_signals.append(col)
+	for col in list(trajectories_neighbors.columns):
+		if is_numeric_dtype(trajectories_neighbors[col]):
+			available_signals.append(col)		
+
+	print('The available signals are : ', available_signals)
+
+	f = open(model_config_path)
+	config = json.load(f)
+	required_signals = config["channels"]
+
+	try:
+		label = config['label']
+		if label=='':
+			label = None
+	except:
+		label = None
+
+	if selected_signals is None:
+		selected_signals = []
+		for s in required_signals:
+			pattern_test = [s in a or s==a for a in available_signals]
+			print(f'Pattern test for signal {s}: ', pattern_test)
+			assert np.any(pattern_test),f'No signal matches with the requirements of the model {required_signals}. Please pass the signals manually with the argument selected_signals or add measurements. Abort.'
+			valid_columns = np.array(available_signals)[np.array(pattern_test)]
+			if len(valid_columns)==1:
+				selected_signals.append(valid_columns[0])
+			else:
+				#print(test_number_of_nan(trajectories, valid_columns))
+				print(f'Found several candidate signals: {valid_columns}')
+				for vc in natsorted(valid_columns):
+					if 'circle' in vc:
+						selected_signals.append(vc)
+						break
+				else:
+					selected_signals.append(valid_columns[0])
+				# do something more complicated in case of one to many columns
+				#pass
+	else:
+		assert len(selected_signals)==len(required_signals),f'Mismatch between the number of required signals {required_signals} and the provided signals {selected_signals}... Abort.'
+
+	print(f'The following channels will be passed to the model: {selected_signals}')
+	trajectories_reference_clean = interpolate_nan_properties(trajectories_reference, track_label=reference_groupby_cols)
+	trajectories_neighbors_clean = interpolate_nan_properties(trajectories_neighbors, track_label=neighbor_groupby_cols)
+	trajectories_pairs_clean = interpolate_nan_properties(trajectories_pairs, track_label=pair_groupby_cols)
+	print(f'{trajectories_pairs_clean.columns=}')
+	
+	max_signal_size = int(trajectories_pairs_clean['pair_FRAME'].max()) + 2
+	pair_tracks =  trajectories_pairs_clean.groupby(pair_groupby_cols).size()
+	signals = np.zeros((len(pair_tracks),max_signal_size, len(selected_signals)))
+	print(f'{max_signal_size=} {len(pair_tracks)=} {signals.shape=}')
+	
+	for i,(pair,group) in enumerate(trajectories_pairs_clean.groupby(pair_groupby_cols)):
+
+		if 'pair_position' not in list(trajectories_pairs_clean.columns):
+			pos_mode = False
+			reference_cell = pair[0]; neighbor_cell = pair[1]
+		else:
+			pos_mode = True
+			reference_cell = pair[1]; neighbor_cell = pair[2]; pos = pair[0]
+
+		if pos_mode and 'reference_position' in list(trajectories_reference_clean.columns) and 'neighbor_position' in list(trajectories_neighbors_clean.columns):
+			reference_filter = (trajectories_reference_clean['reference_TRACK_ID']==reference_cell)&(trajectories_reference_clean['reference_position']==pos)
+			neighbor_filter = (trajectories_neighbors_clean['neighbor_TRACK_ID']==neighbor_cell)&(trajectories_neighbors_clean['neighbor_position']==pos)
+		else:
+			reference_filter = trajectories_reference_clean['reference_TRACK_ID']==reference_cell
+			neighbor_filter = trajectories_neighbors_clean['neighbor_TRACK_ID']==neighbor_cell
+
+		pair_frames = group['pair_FRAME'].to_numpy().astype(int)
+
+		for j,col in enumerate(selected_signals):
+			if col.startswith('pair_'):
+				signal = group[col].to_numpy()
+				signals[i,pair_frames,j] = signal
+				signals[i,max(pair_frames):,j] = signal[-1]
+			elif col.startswith('reference_'):
+				signal = trajectories_reference_clean.loc[reference_filter, col].to_numpy()
+				timeline = trajectories_reference_clean.loc[reference_filter, 'reference_FRAME'].to_numpy()
+				signals[i,timeline,j] = signal
+				signals[i,max(timeline):,j] = signal[-1]
+			elif col.startswith('neighbor_'):
+				signal = trajectories_neighbors_clean.loc[neighbor_filter, col].to_numpy()
+				timeline = trajectories_neighbors_clean.loc[neighbor_filter, 'neighbor_FRAME'].to_numpy()
+				signals[i,timeline,j] = signal
+				signals[i,max(timeline):,j] = signal[-1]					
+
+
+	model = SignalDetectionModel(pretrained=complete_path)
+	print('signal shape: ', signals.shape)
+
+	classes = model.predict_class(signals)
+	times_recast = model.predict_time_of_interest(signals)
+
+	if label is None:
+		class_col = 'pair_class'
+		time_col = 'pair_t0'
+		status_col = 'pair_status'
+	else:
+		class_col = 'pair_class_'+label
+		time_col = 'pair_t_'+label
+		status_col = 'pair_status_'+label
+
+	for i,(pair,group) in enumerate(trajectories_pairs.groupby(pair_groupby_cols)):
+		indices = group.index
+		trajectories_pairs.loc[indices,class_col] = classes[i]
+		trajectories_pairs.loc[indices,time_col] = times_recast[i]
+	print('Done.')
+
+	# At the end rename cols again
+	trajectories_pairs = trajectories_pairs.rename(columns=lambda x: x.replace('pair_',''))
+	trajectories_reference = trajectories_pairs.rename(columns=lambda x: x.replace('reference_',''))
+	trajectories_neighbors = trajectories_pairs.rename(columns=lambda x: x.replace('neighbor_',''))
+	invalid_cols = [c for c in list(trajectories_pairs.columns) if c.startswith('Unnamed')]
+	trajectories_pairs = trajectories_pairs.drop(columns=invalid_cols)
+	
+	return trajectories_pairs
 
 class SignalDetectionModel(object):
 
@@ -1258,10 +1648,10 @@ class SignalDetectionModel(object):
 			csv_logger = CSVLogger(os.sep.join([self.model_folder,'log_classifier.csv']), append=True, separator=';')
 			self.cb.append(csv_logger)
 			checkpoint_path = os.sep.join([self.model_folder,"classifier.h5"])
-			cp_callback = ModelCheckpoint(checkpoint_path,monitor="val_iou",mode="max",verbose=1,save_best_only=True,save_weights_only=False,save_freq="epoch")
+			cp_callback = ModelCheckpoint(checkpoint_path, monitor="val_iou",mode="max",verbose=1,save_best_only=True,save_weights_only=False,save_freq="epoch")
 			self.cb.append(cp_callback)
 			
-			callback_stop = EarlyStopping(monitor='val_iou', patience=100)
+			callback_stop = EarlyStopping(monitor='val_iou',mode='max',patience=100)
 			self.cb.append(callback_stop)
 			
 		elif mode=="regressor":
@@ -1278,7 +1668,7 @@ class SignalDetectionModel(object):
 			cp_callback = ModelCheckpoint(checkpoint_path,monitor="val_loss",mode="min",verbose=1,save_best_only=True,save_weights_only=False,save_freq="epoch")
 			self.cb.append(cp_callback)
 			
-			callback_stop = EarlyStopping(monitor='val_loss', patience=200)
+			callback_stop = EarlyStopping(monitor='val_loss', mode='min', patience=200)
 			self.cb.append(callback_stop)            
 		
 		log_dir = self.model_folder+os.sep
@@ -1312,7 +1702,7 @@ class SignalDetectionModel(object):
 		if isinstance(self.list_of_sets[0],str):
 			# Case 1: a list of npy files to be loaded
 			for s in self.list_of_sets:
-				
+
 				signal_dataset = self.load_set(s)
 				selected_signals, max_length = self.find_best_signal_match(signal_dataset)
 				signals_recast, classes, times_of_interest = self.cast_signals_into_training_data(signal_dataset, selected_signals, max_length)
@@ -1416,7 +1806,6 @@ class SignalDetectionModel(object):
 			x_train_aug.append(aug[0])
 			y_time_train_aug.append(aug[1])
 			y_class_train_aug.append(aug[2])
-		print('per class counts ',counts)
 
 		# Save augmented training set
 		self.x_train = np.array(x_train_aug)
@@ -1469,7 +1858,15 @@ class SignalDetectionModel(object):
 			for i in range(self.n_channels):
 				try:
 					# take into account timeline for accurate time regression
-					timeline = signal_dataset[k]['FRAME'].astype(int)
+
+					if selected_signals[i].startswith('pair_'):
+						timeline = signal_dataset[k]['pair_FRAME'].astype(int)
+					elif selected_signals[i].startswith('reference_'):
+						timeline = signal_dataset[k]['reference_FRAME'].astype(int)
+					elif selected_signals[i].startswith('neighbor_'):
+						timeline = signal_dataset[k]['neighbor_FRAME'].astype(int)
+					else:
+						timeline = signal_dataset[k]['FRAME'].astype(int)
 					signals_recast[k,timeline,i] = signal_dataset[k][selected_signals[i]]
 				except:
 					print(f"Attribute {selected_signals[i]} matched to {self.channel_option[i]} not found in annotation...")

celldetective/tracking.py

@@ -130,7 +130,7 @@ def track(labels, configuration=None, stack=None, spatial_calibration=1, feature
 			tracking_updates = ["motion"]
 		
 		tracker.append(new_btrack_objects)
-		tracker.volume = ((0,volume[0]), (0,volume[1])) #(-1e5, 1e5)
+		tracker.volume = ((0,volume[0]), (0,volume[1]), (-1e5, 1e5)) #(-1e5, 1e5)
 		#print(tracker.volume)
 		tracker.track(tracking_updates=tracking_updates, **track_kwargs)
 		tracker.optimize(options=optimizer_options)
@@ -138,7 +138,11 @@ def track(labels, configuration=None, stack=None, spatial_calibration=1, feature
 		data, properties, graph = tracker.to_napari() #ndim=2
 
 	# do the table post processing and napari options
-	df = pd.DataFrame(data, columns=[column_labels['track'],column_labels['time'],column_labels['y'],column_labels['x']])
+	if data.shape[1]==4:
+		df = pd.DataFrame(data, columns=[column_labels['track'],column_labels['time'],column_labels['y'],column_labels['x']])
+	elif data.shape[1]==5:
+		df = pd.DataFrame(data, columns=[column_labels['track'],column_labels['time'],"z",column_labels['y'],column_labels['x']])
+		df = df.drop(columns=['z'])	
 	df[column_labels['x']+'_um'] = df[column_labels['x']]*spatial_calibration
 	df[column_labels['y']+'_um'] = df[column_labels['y']]*spatial_calibration
 
@@ -160,6 +164,8 @@ def track(labels, configuration=None, stack=None, spatial_calibration=1, feature
 	if clean_trajectories_kwargs is not None:
 		df = clean_trajectories(df.copy(),**clean_trajectories_kwargs)
 
+	df['ID'] = np.arange(len(df)).astype(int)
+
 	if view_on_napari:
 		view_on_napari_btrack(data,properties,graph,stack=stack,labels=labels,relabel=True)